| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 34, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "debug/policy_chosen_logits": 29.054555892944336, | |
| "debug/policy_chosen_logps": -394.62103271484375, | |
| "debug/policy_rejected_logits": 30.527902603149414, | |
| "debug/policy_rejected_logps": -401.28692626953125, | |
| "debug/reference_chosen_logps": -394.62103271484375, | |
| "debug/reference_rejected_logps": -401.28692626953125, | |
| "epoch": 0.029411764705882353, | |
| "grad_norm": 5.461222893396727, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.054555892944336, | |
| "logits/rejected": 30.527902603149414, | |
| "logps/chosen": -394.62103271484375, | |
| "logps/rejected": -401.28692626953125, | |
| "loss": 0.5, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 26.592811584472656, | |
| "debug/policy_chosen_logps": -404.9052429199219, | |
| "debug/policy_rejected_logits": 23.82817268371582, | |
| "debug/policy_rejected_logps": -377.16888427734375, | |
| "debug/reference_chosen_logps": -405.39239501953125, | |
| "debug/reference_rejected_logps": -377.3977966308594, | |
| "epoch": 0.058823529411764705, | |
| "grad_norm": 5.26155447031347, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 26.592811584472656, | |
| "logits/rejected": 23.82817268371582, | |
| "logps/chosen": -404.9052429199219, | |
| "logps/rejected": -377.16888427734375, | |
| "loss": 0.4996, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.004871406592428684, | |
| "rewards/margins": 0.002582397311925888, | |
| "rewards/rejected": 0.002289008814841509, | |
| "step": 2 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.616479873657227, | |
| "debug/policy_chosen_logps": -402.9261474609375, | |
| "debug/policy_rejected_logits": 27.229713439941406, | |
| "debug/policy_rejected_logps": -402.63165283203125, | |
| "debug/reference_chosen_logps": -402.37353515625, | |
| "debug/reference_rejected_logps": -403.0528564453125, | |
| "epoch": 0.08823529411764706, | |
| "grad_norm": 5.686734369927699, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.616479873657227, | |
| "logits/rejected": 27.229713439941406, | |
| "logps/chosen": -402.9261474609375, | |
| "logps/rejected": -402.63165283203125, | |
| "loss": 0.5037, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.005526161286979914, | |
| "rewards/margins": -0.009738006629049778, | |
| "rewards/rejected": 0.004211845342069864, | |
| "step": 3 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.472673416137695, | |
| "debug/policy_chosen_logps": -406.1566467285156, | |
| "debug/policy_rejected_logits": 28.92209815979004, | |
| "debug/policy_rejected_logps": -408.373046875, | |
| "debug/reference_chosen_logps": -406.66375732421875, | |
| "debug/reference_rejected_logps": -407.8741760253906, | |
| "epoch": 0.11764705882352941, | |
| "grad_norm": 5.378959566289513, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.472673416137695, | |
| "logits/rejected": 28.92209815979004, | |
| "logps/chosen": -406.1566467285156, | |
| "logps/rejected": -408.373046875, | |
| "loss": 0.5029, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.005071220453828573, | |
| "rewards/margins": 0.010060157626867294, | |
| "rewards/rejected": -0.004988937638700008, | |
| "step": 4 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 31.492525100708008, | |
| "debug/policy_chosen_logps": -403.90069580078125, | |
| "debug/policy_rejected_logits": 30.92120361328125, | |
| "debug/policy_rejected_logps": -411.68096923828125, | |
| "debug/reference_chosen_logps": -404.43756103515625, | |
| "debug/reference_rejected_logps": -412.2107849121094, | |
| "epoch": 0.14705882352941177, | |
| "grad_norm": 5.270120936652155, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 31.492525100708008, | |
| "logits/rejected": 30.92120361328125, | |
| "logps/chosen": -403.90069580078125, | |
| "logps/rejected": -411.68096923828125, | |
| "loss": 0.4989, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.005368461366742849, | |
| "rewards/margins": 7.053330773487687e-05, | |
| "rewards/rejected": 0.005297927651554346, | |
| "step": 5 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 31.689584732055664, | |
| "debug/policy_chosen_logps": -426.2255554199219, | |
| "debug/policy_rejected_logits": 31.215805053710938, | |
| "debug/policy_rejected_logps": -414.5820007324219, | |
| "debug/reference_chosen_logps": -426.4078369140625, | |
| "debug/reference_rejected_logps": -414.718017578125, | |
| "epoch": 0.17647058823529413, | |
| "grad_norm": 5.511863687507664, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 31.689584732055664, | |
| "logits/rejected": 31.215805053710938, | |
| "logps/chosen": -426.2255554199219, | |
| "logps/rejected": -414.5820007324219, | |
| "loss": 0.5044, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.001822624122723937, | |
| "rewards/margins": 0.0004624177236109972, | |
| "rewards/rejected": 0.0013602066319435835, | |
| "step": 6 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 27.33298683166504, | |
| "debug/policy_chosen_logps": -410.829345703125, | |
| "debug/policy_rejected_logits": 26.893630981445312, | |
| "debug/policy_rejected_logps": -394.69989013671875, | |
| "debug/reference_chosen_logps": -410.4607849121094, | |
| "debug/reference_rejected_logps": -395.7179870605469, | |
| "epoch": 0.20588235294117646, | |
| "grad_norm": 5.290456967833764, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 27.33298683166504, | |
| "logits/rejected": 26.893630981445312, | |
| "logps/chosen": -410.829345703125, | |
| "logps/rejected": -394.69989013671875, | |
| "loss": 0.501, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.003685684408992529, | |
| "rewards/margins": -0.013866348192095757, | |
| "rewards/rejected": 0.010180664248764515, | |
| "step": 7 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 26.145978927612305, | |
| "debug/policy_chosen_logps": -409.4605712890625, | |
| "debug/policy_rejected_logits": 27.502052307128906, | |
| "debug/policy_rejected_logps": -401.6300048828125, | |
| "debug/reference_chosen_logps": -409.36480712890625, | |
| "debug/reference_rejected_logps": -402.3456726074219, | |
| "epoch": 0.23529411764705882, | |
| "grad_norm": 5.259206650868919, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 26.145978927612305, | |
| "logits/rejected": 27.502052307128906, | |
| "logps/chosen": -409.4605712890625, | |
| "logps/rejected": -401.6300048828125, | |
| "loss": 0.4965, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.0009576798183843493, | |
| "rewards/margins": -0.00811409018933773, | |
| "rewards/rejected": 0.007156409788876772, | |
| "step": 8 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.184545516967773, | |
| "debug/policy_chosen_logps": -393.89373779296875, | |
| "debug/policy_rejected_logits": 28.960050582885742, | |
| "debug/policy_rejected_logps": -382.4853820800781, | |
| "debug/reference_chosen_logps": -393.92431640625, | |
| "debug/reference_rejected_logps": -382.33984375, | |
| "epoch": 0.2647058823529412, | |
| "grad_norm": 5.662262990371031, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.184545516967773, | |
| "logits/rejected": 28.960050582885742, | |
| "logps/chosen": -393.89373779296875, | |
| "logps/rejected": -382.4853820800781, | |
| "loss": 0.5007, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0003057096619158983, | |
| "rewards/margins": 0.001760864513926208, | |
| "rewards/rejected": -0.0014551542699337006, | |
| "step": 9 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 24.076282501220703, | |
| "debug/policy_chosen_logps": -394.2970275878906, | |
| "debug/policy_rejected_logits": 21.97032928466797, | |
| "debug/policy_rejected_logps": -381.5108947753906, | |
| "debug/reference_chosen_logps": -393.9175720214844, | |
| "debug/reference_rejected_logps": -381.532470703125, | |
| "epoch": 0.29411764705882354, | |
| "grad_norm": 5.055283348007549, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 24.076282501220703, | |
| "logits/rejected": 21.97032928466797, | |
| "logps/chosen": -394.2970275878906, | |
| "logps/rejected": -381.5108947753906, | |
| "loss": 0.4978, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0037943648640066385, | |
| "rewards/margins": -0.004009971860796213, | |
| "rewards/rejected": 0.0002156068803742528, | |
| "step": 10 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 30.054981231689453, | |
| "debug/policy_chosen_logps": -386.2349853515625, | |
| "debug/policy_rejected_logits": 29.39519691467285, | |
| "debug/policy_rejected_logps": -412.88177490234375, | |
| "debug/reference_chosen_logps": -386.55413818359375, | |
| "debug/reference_rejected_logps": -413.00701904296875, | |
| "epoch": 0.3235294117647059, | |
| "grad_norm": 5.392953865442379, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 30.054981231689453, | |
| "logits/rejected": 29.39519691467285, | |
| "logps/chosen": -386.2349853515625, | |
| "logps/rejected": -412.88177490234375, | |
| "loss": 0.4972, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.0031916426960378885, | |
| "rewards/margins": 0.0019394302507862449, | |
| "rewards/rejected": 0.0012522124452516437, | |
| "step": 11 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 27.566011428833008, | |
| "debug/policy_chosen_logps": -374.5032653808594, | |
| "debug/policy_rejected_logits": 29.68182373046875, | |
| "debug/policy_rejected_logps": -401.15179443359375, | |
| "debug/reference_chosen_logps": -374.087646484375, | |
| "debug/reference_rejected_logps": -402.13214111328125, | |
| "epoch": 0.35294117647058826, | |
| "grad_norm": 6.451284347500255, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 27.566011428833008, | |
| "logits/rejected": 29.68182373046875, | |
| "logps/chosen": -374.5032653808594, | |
| "logps/rejected": -401.15179443359375, | |
| "loss": 0.4998, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.0041561126708984375, | |
| "rewards/margins": -0.013959312811493874, | |
| "rewards/rejected": 0.009803200140595436, | |
| "step": 12 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.40235137939453, | |
| "debug/policy_chosen_logps": -399.9848327636719, | |
| "debug/policy_rejected_logits": 29.182313919067383, | |
| "debug/policy_rejected_logps": -415.01519775390625, | |
| "debug/reference_chosen_logps": -400.4940185546875, | |
| "debug/reference_rejected_logps": -414.45550537109375, | |
| "epoch": 0.38235294117647056, | |
| "grad_norm": 4.8489563556671005, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.40235137939453, | |
| "logits/rejected": 29.182313919067383, | |
| "logps/chosen": -399.9848327636719, | |
| "logps/rejected": -415.01519775390625, | |
| "loss": 0.4979, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.005092009902000427, | |
| "rewards/margins": 0.010688590817153454, | |
| "rewards/rejected": -0.005596580449491739, | |
| "step": 13 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 21.881561279296875, | |
| "debug/policy_chosen_logps": -402.83416748046875, | |
| "debug/policy_rejected_logits": 23.709701538085938, | |
| "debug/policy_rejected_logps": -417.4763488769531, | |
| "debug/reference_chosen_logps": -402.1183166503906, | |
| "debug/reference_rejected_logps": -417.23797607421875, | |
| "epoch": 0.4117647058823529, | |
| "grad_norm": 5.498578228190857, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 21.881561279296875, | |
| "logits/rejected": 23.709701538085938, | |
| "logps/chosen": -402.83416748046875, | |
| "logps/rejected": -417.4763488769531, | |
| "loss": 0.5019, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.007158737629652023, | |
| "rewards/margins": -0.004775161389261484, | |
| "rewards/rejected": -0.0023835753090679646, | |
| "step": 14 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.381258010864258, | |
| "debug/policy_chosen_logps": -395.54803466796875, | |
| "debug/policy_rejected_logits": 28.470420837402344, | |
| "debug/policy_rejected_logps": -387.3695373535156, | |
| "debug/reference_chosen_logps": -395.508056640625, | |
| "debug/reference_rejected_logps": -386.67974853515625, | |
| "epoch": 0.4411764705882353, | |
| "grad_norm": 6.635697402918325, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.381258010864258, | |
| "logits/rejected": 28.470420837402344, | |
| "logps/chosen": -395.54803466796875, | |
| "logps/rejected": -387.3695373535156, | |
| "loss": 0.4963, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0003996272571384907, | |
| "rewards/margins": 0.006498145870864391, | |
| "rewards/rejected": -0.006897774059325457, | |
| "step": 15 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 27.256446838378906, | |
| "debug/policy_chosen_logps": -403.69305419921875, | |
| "debug/policy_rejected_logits": 26.9769344329834, | |
| "debug/policy_rejected_logps": -387.86083984375, | |
| "debug/reference_chosen_logps": -404.33697509765625, | |
| "debug/reference_rejected_logps": -388.44683837890625, | |
| "epoch": 0.47058823529411764, | |
| "grad_norm": 5.539928310976471, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 27.256446838378906, | |
| "logits/rejected": 26.9769344329834, | |
| "logps/chosen": -403.69305419921875, | |
| "logps/rejected": -387.86083984375, | |
| "loss": 0.497, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.006439666263759136, | |
| "rewards/margins": 0.0005795289762318134, | |
| "rewards/rejected": 0.00586013775318861, | |
| "step": 16 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 27.54875373840332, | |
| "debug/policy_chosen_logps": -379.23291015625, | |
| "debug/policy_rejected_logits": 28.37594985961914, | |
| "debug/policy_rejected_logps": -391.45654296875, | |
| "debug/reference_chosen_logps": -380.18060302734375, | |
| "debug/reference_rejected_logps": -390.98968505859375, | |
| "epoch": 0.5, | |
| "grad_norm": 5.2957197578010415, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 27.54875373840332, | |
| "logits/rejected": 28.37594985961914, | |
| "logps/chosen": -379.23291015625, | |
| "logps/rejected": -391.45654296875, | |
| "loss": 0.4938, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.009477118961513042, | |
| "rewards/margins": 0.01414569839835167, | |
| "rewards/rejected": -0.0046685789711773396, | |
| "step": 17 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 25.160919189453125, | |
| "debug/policy_chosen_logps": -389.9479675292969, | |
| "debug/policy_rejected_logits": 25.0775146484375, | |
| "debug/policy_rejected_logps": -372.14349365234375, | |
| "debug/reference_chosen_logps": -389.97222900390625, | |
| "debug/reference_rejected_logps": -371.96075439453125, | |
| "epoch": 0.5294117647058824, | |
| "grad_norm": 4.954808294891358, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 25.160919189453125, | |
| "logits/rejected": 25.0775146484375, | |
| "logps/chosen": -389.9479675292969, | |
| "logps/rejected": -372.14349365234375, | |
| "loss": 0.4953, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.00024272920563817024, | |
| "rewards/margins": 0.002069931011646986, | |
| "rewards/rejected": -0.0018272020388394594, | |
| "step": 18 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 25.586524963378906, | |
| "debug/policy_chosen_logps": -374.25006103515625, | |
| "debug/policy_rejected_logits": 28.893770217895508, | |
| "debug/policy_rejected_logps": -412.3420104980469, | |
| "debug/reference_chosen_logps": -374.282958984375, | |
| "debug/reference_rejected_logps": -411.41131591796875, | |
| "epoch": 0.5588235294117647, | |
| "grad_norm": 4.9850918433013724, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 25.586524963378906, | |
| "logits/rejected": 28.893770217895508, | |
| "logps/chosen": -374.25006103515625, | |
| "logps/rejected": -412.3420104980469, | |
| "loss": 0.5002, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.0003293610643595457, | |
| "rewards/margins": 0.009636610746383667, | |
| "rewards/rejected": -0.009307250380516052, | |
| "step": 19 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 24.718896865844727, | |
| "debug/policy_chosen_logps": -363.05120849609375, | |
| "debug/policy_rejected_logits": 24.986190795898438, | |
| "debug/policy_rejected_logps": -388.0316162109375, | |
| "debug/reference_chosen_logps": -362.204833984375, | |
| "debug/reference_rejected_logps": -386.9569091796875, | |
| "epoch": 0.5882352941176471, | |
| "grad_norm": 5.05147280128332, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 24.718896865844727, | |
| "logits/rejected": 24.986190795898438, | |
| "logps/chosen": -363.05120849609375, | |
| "logps/rejected": -388.0316162109375, | |
| "loss": 0.4997, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.00846382137387991, | |
| "rewards/margins": 0.0022830581292510033, | |
| "rewards/rejected": -0.010746879503130913, | |
| "step": 20 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.18993377685547, | |
| "debug/policy_chosen_logps": -397.7740478515625, | |
| "debug/policy_rejected_logits": 23.599760055541992, | |
| "debug/policy_rejected_logps": -400.73492431640625, | |
| "debug/reference_chosen_logps": -399.1761169433594, | |
| "debug/reference_rejected_logps": -399.78948974609375, | |
| "epoch": 0.6176470588235294, | |
| "grad_norm": 6.701576850276441, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.18993377685547, | |
| "logits/rejected": 23.599760055541992, | |
| "logps/chosen": -397.7740478515625, | |
| "logps/rejected": -400.73492431640625, | |
| "loss": 0.4981, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.014020653441548347, | |
| "rewards/margins": 0.023475077003240585, | |
| "rewards/rejected": -0.009454423561692238, | |
| "step": 21 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.8790283203125, | |
| "debug/policy_chosen_logps": -413.7144775390625, | |
| "debug/policy_rejected_logits": 28.82781982421875, | |
| "debug/policy_rejected_logps": -409.18292236328125, | |
| "debug/reference_chosen_logps": -414.69781494140625, | |
| "debug/reference_rejected_logps": -407.171142578125, | |
| "epoch": 0.6470588235294118, | |
| "grad_norm": 6.323947957916722, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.8790283203125, | |
| "logits/rejected": 28.82781982421875, | |
| "logps/chosen": -413.7144775390625, | |
| "logps/rejected": -409.18292236328125, | |
| "loss": 0.4999, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.009833297692239285, | |
| "rewards/margins": 0.029951322823762894, | |
| "rewards/rejected": -0.020118026062846184, | |
| "step": 22 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 27.423133850097656, | |
| "debug/policy_chosen_logps": -377.9825439453125, | |
| "debug/policy_rejected_logits": 29.817766189575195, | |
| "debug/policy_rejected_logps": -400.9486389160156, | |
| "debug/reference_chosen_logps": -378.6519775390625, | |
| "debug/reference_rejected_logps": -400.3038024902344, | |
| "epoch": 0.6764705882352942, | |
| "grad_norm": 7.650698869625993, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 27.423133850097656, | |
| "logits/rejected": 29.817766189575195, | |
| "logps/chosen": -377.9825439453125, | |
| "logps/rejected": -400.9486389160156, | |
| "loss": 0.4944, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.006694145500659943, | |
| "rewards/margins": 0.013142585754394531, | |
| "rewards/rejected": -0.006448440253734589, | |
| "step": 23 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 26.779478073120117, | |
| "debug/policy_chosen_logps": -392.8223571777344, | |
| "debug/policy_rejected_logits": 28.28175163269043, | |
| "debug/policy_rejected_logps": -394.74676513671875, | |
| "debug/reference_chosen_logps": -393.3936462402344, | |
| "debug/reference_rejected_logps": -394.11358642578125, | |
| "epoch": 0.7058823529411765, | |
| "grad_norm": 5.251742483181524, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 26.779478073120117, | |
| "logits/rejected": 28.28175163269043, | |
| "logps/chosen": -392.8223571777344, | |
| "logps/rejected": -394.74676513671875, | |
| "loss": 0.5016, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.005712966434657574, | |
| "rewards/margins": 0.012044791132211685, | |
| "rewards/rejected": -0.0063318246975541115, | |
| "step": 24 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.304908752441406, | |
| "debug/policy_chosen_logps": -395.7744445800781, | |
| "debug/policy_rejected_logits": 27.35923957824707, | |
| "debug/policy_rejected_logps": -403.1441955566406, | |
| "debug/reference_chosen_logps": -395.40789794921875, | |
| "debug/reference_rejected_logps": -403.5139465332031, | |
| "epoch": 0.7352941176470589, | |
| "grad_norm": 4.968117146678153, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.304908752441406, | |
| "logits/rejected": 27.35923957824707, | |
| "logps/chosen": -395.7744445800781, | |
| "logps/rejected": -403.1441955566406, | |
| "loss": 0.4985, | |
| "rewards/accuracies": 0.125, | |
| "rewards/chosen": -0.003665314055979252, | |
| "rewards/margins": -0.007362784817814827, | |
| "rewards/rejected": 0.0036974712274968624, | |
| "step": 25 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.73921012878418, | |
| "debug/policy_chosen_logps": -396.11834716796875, | |
| "debug/policy_rejected_logits": 28.473979949951172, | |
| "debug/policy_rejected_logps": -419.5787353515625, | |
| "debug/reference_chosen_logps": -396.2462158203125, | |
| "debug/reference_rejected_logps": -418.2436218261719, | |
| "epoch": 0.7647058823529411, | |
| "grad_norm": 5.07020088847226, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.73921012878418, | |
| "logits/rejected": 28.473979949951172, | |
| "logps/chosen": -396.11834716796875, | |
| "logps/rejected": -419.5787353515625, | |
| "loss": 0.4994, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.0012784956488758326, | |
| "rewards/margins": 0.014629820361733437, | |
| "rewards/rejected": -0.013351325877010822, | |
| "step": 26 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 30.456932067871094, | |
| "debug/policy_chosen_logps": -410.0908203125, | |
| "debug/policy_rejected_logits": 28.822614669799805, | |
| "debug/policy_rejected_logps": -418.9249572753906, | |
| "debug/reference_chosen_logps": -410.21063232421875, | |
| "debug/reference_rejected_logps": -417.00714111328125, | |
| "epoch": 0.7941176470588235, | |
| "grad_norm": 6.10176209535134, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 30.456932067871094, | |
| "logits/rejected": 28.822614669799805, | |
| "logps/chosen": -410.0908203125, | |
| "logps/rejected": -418.9249572753906, | |
| "loss": 0.4853, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.0011978147085756063, | |
| "rewards/margins": 0.020375745370984077, | |
| "rewards/rejected": -0.019177932292222977, | |
| "step": 27 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.178850173950195, | |
| "debug/policy_chosen_logps": -408.07464599609375, | |
| "debug/policy_rejected_logits": 28.366477966308594, | |
| "debug/policy_rejected_logps": -414.43853759765625, | |
| "debug/reference_chosen_logps": -407.51043701171875, | |
| "debug/reference_rejected_logps": -410.44268798828125, | |
| "epoch": 0.8235294117647058, | |
| "grad_norm": 4.919999726189566, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.178850173950195, | |
| "logits/rejected": 28.366477966308594, | |
| "logps/chosen": -408.07464599609375, | |
| "logps/rejected": -414.43853759765625, | |
| "loss": 0.499, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.005642051808536053, | |
| "rewards/margins": 0.03431610018014908, | |
| "rewards/rejected": -0.039958152920007706, | |
| "step": 28 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.156843185424805, | |
| "debug/policy_chosen_logps": -409.95208740234375, | |
| "debug/policy_rejected_logits": 27.127243041992188, | |
| "debug/policy_rejected_logps": -393.4117431640625, | |
| "debug/reference_chosen_logps": -409.63983154296875, | |
| "debug/reference_rejected_logps": -392.2528076171875, | |
| "epoch": 0.8529411764705882, | |
| "grad_norm": 5.038344787361152, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.156843185424805, | |
| "logits/rejected": 27.127243041992188, | |
| "logps/chosen": -409.95208740234375, | |
| "logps/rejected": -393.4117431640625, | |
| "loss": 0.4935, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.003122864291071892, | |
| "rewards/margins": 0.008466909639537334, | |
| "rewards/rejected": -0.0115897748619318, | |
| "step": 29 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 26.275718688964844, | |
| "debug/policy_chosen_logps": -406.03277587890625, | |
| "debug/policy_rejected_logits": 26.794349670410156, | |
| "debug/policy_rejected_logps": -410.08282470703125, | |
| "debug/reference_chosen_logps": -405.848876953125, | |
| "debug/reference_rejected_logps": -410.5152587890625, | |
| "epoch": 0.8823529411764706, | |
| "grad_norm": 6.078237487272278, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 26.275718688964844, | |
| "logits/rejected": 26.794349670410156, | |
| "logps/chosen": -406.03277587890625, | |
| "logps/rejected": -410.08282470703125, | |
| "loss": 0.4941, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0018388747703284025, | |
| "rewards/margins": -0.006163253448903561, | |
| "rewards/rejected": 0.0043243784457445145, | |
| "step": 30 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 27.000301361083984, | |
| "debug/policy_chosen_logps": -391.47613525390625, | |
| "debug/policy_rejected_logits": 29.51080322265625, | |
| "debug/policy_rejected_logps": -386.7709655761719, | |
| "debug/reference_chosen_logps": -390.591552734375, | |
| "debug/reference_rejected_logps": -387.83935546875, | |
| "epoch": 0.9117647058823529, | |
| "grad_norm": 5.608762984875432, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 27.000301361083984, | |
| "logits/rejected": 29.51080322265625, | |
| "logps/chosen": -391.47613525390625, | |
| "logps/rejected": -386.7709655761719, | |
| "loss": 0.4983, | |
| "rewards/accuracies": 0.125, | |
| "rewards/chosen": -0.008846206590533257, | |
| "rewards/margins": -0.019530104473233223, | |
| "rewards/rejected": 0.010683897882699966, | |
| "step": 31 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.469999313354492, | |
| "debug/policy_chosen_logps": -390.48291015625, | |
| "debug/policy_rejected_logits": 28.674427032470703, | |
| "debug/policy_rejected_logps": -414.2558288574219, | |
| "debug/reference_chosen_logps": -392.53704833984375, | |
| "debug/reference_rejected_logps": -416.4234619140625, | |
| "epoch": 0.9411764705882353, | |
| "grad_norm": 5.952413372138852, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.469999313354492, | |
| "logits/rejected": 28.674427032470703, | |
| "logps/chosen": -390.48291015625, | |
| "logps/rejected": -414.2558288574219, | |
| "loss": 0.4928, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.02054138109087944, | |
| "rewards/margins": -0.0011348333209753036, | |
| "rewards/rejected": 0.021676216274499893, | |
| "step": 32 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 30.11175537109375, | |
| "debug/policy_chosen_logps": -393.00079345703125, | |
| "debug/policy_rejected_logits": 31.391122817993164, | |
| "debug/policy_rejected_logps": -419.4772644042969, | |
| "debug/reference_chosen_logps": -393.62738037109375, | |
| "debug/reference_rejected_logps": -416.98974609375, | |
| "epoch": 0.9705882352941176, | |
| "grad_norm": 5.204183724206507, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 30.11175537109375, | |
| "logits/rejected": 31.391122817993164, | |
| "logps/chosen": -393.00079345703125, | |
| "logps/rejected": -419.4772644042969, | |
| "loss": 0.4877, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.006265563890337944, | |
| "rewards/margins": 0.031140554696321487, | |
| "rewards/rejected": -0.024874988943338394, | |
| "step": 33 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.005823135375977, | |
| "debug/policy_chosen_logps": -417.3971252441406, | |
| "debug/policy_rejected_logits": 24.94090461730957, | |
| "debug/policy_rejected_logps": -404.98779296875, | |
| "debug/reference_chosen_logps": -416.21746826171875, | |
| "debug/reference_rejected_logps": -404.94940185546875, | |
| "epoch": 1.0, | |
| "grad_norm": 5.733599790823146, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.005823135375977, | |
| "logits/rejected": 24.94090461730957, | |
| "logps/chosen": -417.3971252441406, | |
| "logps/rejected": -404.98779296875, | |
| "loss": 0.4673, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.011796417646110058, | |
| "rewards/margins": -0.01141277328133583, | |
| "rewards/rejected": -0.00038364529609680176, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 34, | |
| "total_flos": 0.0, | |
| "train_loss": 0.4969010160249822, | |
| "train_runtime": 407.6559, | |
| "train_samples_per_second": 5.247, | |
| "train_steps_per_second": 0.083 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 34, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |