| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 80, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "debug/policy_chosen_logits": 16.754077911376953, | |
| "debug/policy_chosen_logps": -410.5740966796875, | |
| "debug/policy_rejected_logits": 27.903383255004883, | |
| "debug/policy_rejected_logps": -454.6379699707031, | |
| "debug/reference_chosen_logps": -410.5740966796875, | |
| "debug/reference_rejected_logps": -454.6379699707031, | |
| "epoch": 0.0125, | |
| "grad_norm": 8.96581893169365, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 16.754077911376953, | |
| "logits/rejected": 27.903383255004883, | |
| "logps/chosen": -410.5740966796875, | |
| "logps/rejected": -454.6379699707031, | |
| "loss": 0.5, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 21.365768432617188, | |
| "debug/policy_chosen_logps": -414.0776062011719, | |
| "debug/policy_rejected_logits": 24.07695770263672, | |
| "debug/policy_rejected_logps": -463.8160095214844, | |
| "debug/reference_chosen_logps": -414.4972839355469, | |
| "debug/reference_rejected_logps": -464.1080322265625, | |
| "epoch": 0.025, | |
| "grad_norm": 8.150809103935691, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 21.365768432617188, | |
| "logits/rejected": 24.07695770263672, | |
| "logps/chosen": -414.0776062011719, | |
| "logps/rejected": -463.8160095214844, | |
| "loss": 0.4994, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.004196891561150551, | |
| "rewards/margins": 0.0012764740968123078, | |
| "rewards/rejected": 0.002920417580753565, | |
| "step": 2 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 25.1464786529541, | |
| "debug/policy_chosen_logps": -451.97662353515625, | |
| "debug/policy_rejected_logits": 20.219999313354492, | |
| "debug/policy_rejected_logps": -430.0290222167969, | |
| "debug/reference_chosen_logps": -451.04254150390625, | |
| "debug/reference_rejected_logps": -429.7825927734375, | |
| "epoch": 0.0375, | |
| "grad_norm": 7.628476816083828, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 25.1464786529541, | |
| "logits/rejected": 20.219999313354492, | |
| "logps/chosen": -451.97662353515625, | |
| "logps/rejected": -430.0290222167969, | |
| "loss": 0.4977, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.009341010823845863, | |
| "rewards/margins": -0.006876640021800995, | |
| "rewards/rejected": -0.002464370569214225, | |
| "step": 3 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.24092674255371, | |
| "debug/policy_chosen_logps": -438.38348388671875, | |
| "debug/policy_rejected_logits": 26.255001068115234, | |
| "debug/policy_rejected_logps": -463.3494567871094, | |
| "debug/reference_chosen_logps": -437.87542724609375, | |
| "debug/reference_rejected_logps": -461.87445068359375, | |
| "epoch": 0.05, | |
| "grad_norm": 8.325323905761818, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.24092674255371, | |
| "logits/rejected": 26.255001068115234, | |
| "logps/chosen": -438.38348388671875, | |
| "logps/rejected": -463.3494567871094, | |
| "loss": 0.4907, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.0050805662758648396, | |
| "rewards/margins": 0.009669303894042969, | |
| "rewards/rejected": -0.014749869704246521, | |
| "step": 4 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 17.008541107177734, | |
| "debug/policy_chosen_logps": -407.684814453125, | |
| "debug/policy_rejected_logits": 23.88060188293457, | |
| "debug/policy_rejected_logps": -430.1419982910156, | |
| "debug/reference_chosen_logps": -407.943115234375, | |
| "debug/reference_rejected_logps": -429.351806640625, | |
| "epoch": 0.0625, | |
| "grad_norm": 8.68321847373836, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 17.008541107177734, | |
| "logits/rejected": 23.88060188293457, | |
| "logps/chosen": -407.684814453125, | |
| "logps/rejected": -430.1419982910156, | |
| "loss": 0.4865, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.002583122346550226, | |
| "rewards/margins": 0.010485000908374786, | |
| "rewards/rejected": -0.007901879027485847, | |
| "step": 5 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 23.972543716430664, | |
| "debug/policy_chosen_logps": -452.3487548828125, | |
| "debug/policy_rejected_logits": 23.104774475097656, | |
| "debug/policy_rejected_logps": -456.2618408203125, | |
| "debug/reference_chosen_logps": -451.52825927734375, | |
| "debug/reference_rejected_logps": -454.55670166015625, | |
| "epoch": 0.075, | |
| "grad_norm": 6.132910409397078, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 23.972543716430664, | |
| "logits/rejected": 23.104774475097656, | |
| "logps/chosen": -452.3487548828125, | |
| "logps/rejected": -456.2618408203125, | |
| "loss": 0.4884, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.00820488017052412, | |
| "rewards/margins": 0.008846473880112171, | |
| "rewards/rejected": -0.01705135405063629, | |
| "step": 6 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.909212112426758, | |
| "debug/policy_chosen_logps": -402.86126708984375, | |
| "debug/policy_rejected_logits": 26.987449645996094, | |
| "debug/policy_rejected_logps": -445.4053039550781, | |
| "debug/reference_chosen_logps": -403.7894287109375, | |
| "debug/reference_rejected_logps": -444.8185729980469, | |
| "epoch": 0.0875, | |
| "grad_norm": 8.214444111303614, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.909212112426758, | |
| "logits/rejected": 26.987449645996094, | |
| "logps/chosen": -402.86126708984375, | |
| "logps/rejected": -445.4053039550781, | |
| "loss": 0.4736, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.00928169209510088, | |
| "rewards/margins": 0.015148963779211044, | |
| "rewards/rejected": -0.005867272149771452, | |
| "step": 7 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 22.914825439453125, | |
| "debug/policy_chosen_logps": -427.63922119140625, | |
| "debug/policy_rejected_logits": 26.372671127319336, | |
| "debug/policy_rejected_logps": -483.4498291015625, | |
| "debug/reference_chosen_logps": -427.9068603515625, | |
| "debug/reference_rejected_logps": -479.41705322265625, | |
| "epoch": 0.1, | |
| "grad_norm": 8.86676608632659, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 22.914825439453125, | |
| "logits/rejected": 26.372671127319336, | |
| "logps/chosen": -427.63922119140625, | |
| "logps/rejected": -483.4498291015625, | |
| "loss": 0.4702, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.0026765051297843456, | |
| "rewards/margins": 0.04300403222441673, | |
| "rewards/rejected": -0.0403275266289711, | |
| "step": 8 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.00334930419922, | |
| "debug/policy_chosen_logps": -432.1858215332031, | |
| "debug/policy_rejected_logits": 26.359827041625977, | |
| "debug/policy_rejected_logps": -458.9487609863281, | |
| "debug/reference_chosen_logps": -433.03985595703125, | |
| "debug/reference_rejected_logps": -455.154296875, | |
| "epoch": 0.1125, | |
| "grad_norm": 7.71461629791825, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.00334930419922, | |
| "logits/rejected": 26.359827041625977, | |
| "logps/chosen": -432.1858215332031, | |
| "logps/rejected": -458.9487609863281, | |
| "loss": 0.4593, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.008540572598576546, | |
| "rewards/margins": 0.04648509621620178, | |
| "rewards/rejected": -0.037944525480270386, | |
| "step": 9 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.777141571044922, | |
| "debug/policy_chosen_logps": -417.86517333984375, | |
| "debug/policy_rejected_logits": 20.1833553314209, | |
| "debug/policy_rejected_logps": -449.54583740234375, | |
| "debug/reference_chosen_logps": -420.6201171875, | |
| "debug/reference_rejected_logps": -447.97467041015625, | |
| "epoch": 0.125, | |
| "grad_norm": 7.863102292412372, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.777141571044922, | |
| "logits/rejected": 20.1833553314209, | |
| "logps/chosen": -417.86517333984375, | |
| "logps/rejected": -449.54583740234375, | |
| "loss": 0.4364, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.027549132704734802, | |
| "rewards/margins": 0.04326057434082031, | |
| "rewards/rejected": -0.01571143977344036, | |
| "step": 10 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.944578170776367, | |
| "debug/policy_chosen_logps": -417.01837158203125, | |
| "debug/policy_rejected_logits": 25.846759796142578, | |
| "debug/policy_rejected_logps": -450.23468017578125, | |
| "debug/reference_chosen_logps": -422.13604736328125, | |
| "debug/reference_rejected_logps": -443.507568359375, | |
| "epoch": 0.1375, | |
| "grad_norm": 6.8158984104886, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.944578170776367, | |
| "logits/rejected": 25.846759796142578, | |
| "logps/chosen": -417.01837158203125, | |
| "logps/rejected": -450.23468017578125, | |
| "loss": 0.4331, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.05117690563201904, | |
| "rewards/margins": 0.11844829469919205, | |
| "rewards/rejected": -0.067271389067173, | |
| "step": 11 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.338014602661133, | |
| "debug/policy_chosen_logps": -406.2593994140625, | |
| "debug/policy_rejected_logits": 27.694448471069336, | |
| "debug/policy_rejected_logps": -448.88671875, | |
| "debug/reference_chosen_logps": -411.54815673828125, | |
| "debug/reference_rejected_logps": -443.79315185546875, | |
| "epoch": 0.15, | |
| "grad_norm": 5.494973875316365, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.338014602661133, | |
| "logits/rejected": 27.694448471069336, | |
| "logps/chosen": -406.2593994140625, | |
| "logps/rejected": -448.88671875, | |
| "loss": 0.4392, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.05288715288043022, | |
| "rewards/margins": 0.10382324457168579, | |
| "rewards/rejected": -0.05093608796596527, | |
| "step": 12 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 17.940452575683594, | |
| "debug/policy_chosen_logps": -409.26666259765625, | |
| "debug/policy_rejected_logits": 26.594871520996094, | |
| "debug/policy_rejected_logps": -462.8791809082031, | |
| "debug/reference_chosen_logps": -418.971923828125, | |
| "debug/reference_rejected_logps": -455.71661376953125, | |
| "epoch": 0.1625, | |
| "grad_norm": 6.839480437899935, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 17.940452575683594, | |
| "logits/rejected": 26.594871520996094, | |
| "logps/chosen": -409.26666259765625, | |
| "logps/rejected": -462.8791809082031, | |
| "loss": 0.4146, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.09705245494842529, | |
| "rewards/margins": 0.1686779260635376, | |
| "rewards/rejected": -0.0716254711151123, | |
| "step": 13 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 21.89854621887207, | |
| "debug/policy_chosen_logps": -419.85137939453125, | |
| "debug/policy_rejected_logits": 28.44005012512207, | |
| "debug/policy_rejected_logps": -485.2733154296875, | |
| "debug/reference_chosen_logps": -422.12200927734375, | |
| "debug/reference_rejected_logps": -476.89013671875, | |
| "epoch": 0.175, | |
| "grad_norm": 6.253412161075545, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 21.89854621887207, | |
| "logits/rejected": 28.44005012512207, | |
| "logps/chosen": -419.85137939453125, | |
| "logps/rejected": -485.2733154296875, | |
| "loss": 0.4071, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.022706221789121628, | |
| "rewards/margins": 0.10653793066740036, | |
| "rewards/rejected": -0.08383171260356903, | |
| "step": 14 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 17.64580726623535, | |
| "debug/policy_chosen_logps": -411.6397705078125, | |
| "debug/policy_rejected_logits": 21.619295120239258, | |
| "debug/policy_rejected_logps": -427.70172119140625, | |
| "debug/reference_chosen_logps": -417.8538818359375, | |
| "debug/reference_rejected_logps": -421.78173828125, | |
| "epoch": 0.1875, | |
| "grad_norm": 5.383646265219509, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 17.64580726623535, | |
| "logits/rejected": 21.619295120239258, | |
| "logps/chosen": -411.6397705078125, | |
| "logps/rejected": -427.70172119140625, | |
| "loss": 0.4213, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.0621412992477417, | |
| "rewards/margins": 0.12134108692407608, | |
| "rewards/rejected": -0.05919978767633438, | |
| "step": 15 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.415315628051758, | |
| "debug/policy_chosen_logps": -423.58721923828125, | |
| "debug/policy_rejected_logits": 28.420116424560547, | |
| "debug/policy_rejected_logps": -477.6827697753906, | |
| "debug/reference_chosen_logps": -427.49700927734375, | |
| "debug/reference_rejected_logps": -467.93035888671875, | |
| "epoch": 0.2, | |
| "grad_norm": 5.208847435610174, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.415315628051758, | |
| "logits/rejected": 28.420116424560547, | |
| "logps/chosen": -423.58721923828125, | |
| "logps/rejected": -477.6827697753906, | |
| "loss": 0.4097, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.03909797593951225, | |
| "rewards/margins": 0.1366221308708191, | |
| "rewards/rejected": -0.09752415120601654, | |
| "step": 16 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 17.148258209228516, | |
| "debug/policy_chosen_logps": -400.81634521484375, | |
| "debug/policy_rejected_logits": 23.263225555419922, | |
| "debug/policy_rejected_logps": -491.70166015625, | |
| "debug/reference_chosen_logps": -413.73779296875, | |
| "debug/reference_rejected_logps": -475.34991455078125, | |
| "epoch": 0.2125, | |
| "grad_norm": 6.371009448433531, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 17.148258209228516, | |
| "logits/rejected": 23.263225555419922, | |
| "logps/chosen": -400.81634521484375, | |
| "logps/rejected": -491.70166015625, | |
| "loss": 0.362, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.12921428680419922, | |
| "rewards/margins": 0.2927318513393402, | |
| "rewards/rejected": -0.163517564535141, | |
| "step": 17 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.67523956298828, | |
| "debug/policy_chosen_logps": -417.3398742675781, | |
| "debug/policy_rejected_logits": 28.897300720214844, | |
| "debug/policy_rejected_logps": -495.4722900390625, | |
| "debug/reference_chosen_logps": -429.54852294921875, | |
| "debug/reference_rejected_logps": -480.121337890625, | |
| "epoch": 0.225, | |
| "grad_norm": 4.6720603095118305, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.67523956298828, | |
| "logits/rejected": 28.897300720214844, | |
| "logps/chosen": -417.3398742675781, | |
| "logps/rejected": -495.4722900390625, | |
| "loss": 0.3625, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.12208674848079681, | |
| "rewards/margins": 0.2755962014198303, | |
| "rewards/rejected": -0.1535094678401947, | |
| "step": 18 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 22.884260177612305, | |
| "debug/policy_chosen_logps": -439.2691650390625, | |
| "debug/policy_rejected_logits": 24.13343620300293, | |
| "debug/policy_rejected_logps": -466.717041015625, | |
| "debug/reference_chosen_logps": -442.8848876953125, | |
| "debug/reference_rejected_logps": -443.7426452636719, | |
| "epoch": 0.2375, | |
| "grad_norm": 6.3874571813001, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 22.884260177612305, | |
| "logits/rejected": 24.13343620300293, | |
| "logps/chosen": -439.2691650390625, | |
| "logps/rejected": -466.717041015625, | |
| "loss": 0.2871, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.03615710884332657, | |
| "rewards/margins": 0.26590147614479065, | |
| "rewards/rejected": -0.22974437475204468, | |
| "step": 19 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.085355758666992, | |
| "debug/policy_chosen_logps": -405.03131103515625, | |
| "debug/policy_rejected_logits": 25.65454864501953, | |
| "debug/policy_rejected_logps": -455.36871337890625, | |
| "debug/reference_chosen_logps": -411.85076904296875, | |
| "debug/reference_rejected_logps": -429.8324890136719, | |
| "epoch": 0.25, | |
| "grad_norm": 4.779696568165662, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.085355758666992, | |
| "logits/rejected": 25.65454864501953, | |
| "logps/chosen": -405.03131103515625, | |
| "logps/rejected": -455.36871337890625, | |
| "loss": 0.3732, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.06819470226764679, | |
| "rewards/margins": 0.3235568404197693, | |
| "rewards/rejected": -0.2553621232509613, | |
| "step": 20 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.691024780273438, | |
| "debug/policy_chosen_logps": -395.2339782714844, | |
| "debug/policy_rejected_logits": 28.38985824584961, | |
| "debug/policy_rejected_logps": -447.9287109375, | |
| "debug/reference_chosen_logps": -419.5431823730469, | |
| "debug/reference_rejected_logps": -440.24517822265625, | |
| "epoch": 0.2625, | |
| "grad_norm": 4.138890165784992, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.691024780273438, | |
| "logits/rejected": 28.38985824584961, | |
| "logps/chosen": -395.2339782714844, | |
| "logps/rejected": -447.9287109375, | |
| "loss": 0.3061, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.24309200048446655, | |
| "rewards/margins": 0.3199271261692047, | |
| "rewards/rejected": -0.07683513313531876, | |
| "step": 21 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.216005325317383, | |
| "debug/policy_chosen_logps": -398.2820129394531, | |
| "debug/policy_rejected_logits": 27.54266929626465, | |
| "debug/policy_rejected_logps": -462.150146484375, | |
| "debug/reference_chosen_logps": -406.9403076171875, | |
| "debug/reference_rejected_logps": -450.525390625, | |
| "epoch": 0.275, | |
| "grad_norm": 4.456237735720388, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.216005325317383, | |
| "logits/rejected": 27.54266929626465, | |
| "logps/chosen": -398.2820129394531, | |
| "logps/rejected": -462.150146484375, | |
| "loss": 0.3342, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.08658306300640106, | |
| "rewards/margins": 0.20283077657222748, | |
| "rewards/rejected": -0.11624770611524582, | |
| "step": 22 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.20200538635254, | |
| "debug/policy_chosen_logps": -399.0750427246094, | |
| "debug/policy_rejected_logits": 29.885364532470703, | |
| "debug/policy_rejected_logps": -464.96728515625, | |
| "debug/reference_chosen_logps": -416.9361877441406, | |
| "debug/reference_rejected_logps": -453.67095947265625, | |
| "epoch": 0.2875, | |
| "grad_norm": 6.33340244824422, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.20200538635254, | |
| "logits/rejected": 29.885364532470703, | |
| "logps/chosen": -399.0750427246094, | |
| "logps/rejected": -464.96728515625, | |
| "loss": 0.3919, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.17861154675483704, | |
| "rewards/margins": 0.2915750741958618, | |
| "rewards/rejected": -0.11296352744102478, | |
| "step": 23 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.623193740844727, | |
| "debug/policy_chosen_logps": -405.6262512207031, | |
| "debug/policy_rejected_logits": 30.310110092163086, | |
| "debug/policy_rejected_logps": -461.8826599121094, | |
| "debug/reference_chosen_logps": -436.78277587890625, | |
| "debug/reference_rejected_logps": -433.28759765625, | |
| "epoch": 0.3, | |
| "grad_norm": 6.894047386241825, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.623193740844727, | |
| "logits/rejected": 30.310110092163086, | |
| "logps/chosen": -405.6262512207031, | |
| "logps/rejected": -461.8826599121094, | |
| "loss": 0.3025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3115653395652771, | |
| "rewards/margins": 0.5975160598754883, | |
| "rewards/rejected": -0.2859506905078888, | |
| "step": 24 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 21.141868591308594, | |
| "debug/policy_chosen_logps": -387.9573974609375, | |
| "debug/policy_rejected_logits": 25.790800094604492, | |
| "debug/policy_rejected_logps": -441.328369140625, | |
| "debug/reference_chosen_logps": -414.5980529785156, | |
| "debug/reference_rejected_logps": -434.82177734375, | |
| "epoch": 0.3125, | |
| "grad_norm": 4.7433049140532395, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 21.141868591308594, | |
| "logits/rejected": 25.790800094604492, | |
| "logps/chosen": -387.9573974609375, | |
| "logps/rejected": -441.328369140625, | |
| "loss": 0.3551, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.26640671491622925, | |
| "rewards/margins": 0.33147239685058594, | |
| "rewards/rejected": -0.0650656521320343, | |
| "step": 25 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.596500396728516, | |
| "debug/policy_chosen_logps": -389.6454162597656, | |
| "debug/policy_rejected_logits": 27.930261611938477, | |
| "debug/policy_rejected_logps": -470.5025329589844, | |
| "debug/reference_chosen_logps": -414.76007080078125, | |
| "debug/reference_rejected_logps": -440.7433166503906, | |
| "epoch": 0.325, | |
| "grad_norm": 4.306188553423187, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.596500396728516, | |
| "logits/rejected": 27.930261611938477, | |
| "logps/chosen": -389.6454162597656, | |
| "logps/rejected": -470.5025329589844, | |
| "loss": 0.2747, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.25114673376083374, | |
| "rewards/margins": 0.5487388968467712, | |
| "rewards/rejected": -0.2975921928882599, | |
| "step": 26 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.838258743286133, | |
| "debug/policy_chosen_logps": -396.57708740234375, | |
| "debug/policy_rejected_logits": 27.663278579711914, | |
| "debug/policy_rejected_logps": -454.4193115234375, | |
| "debug/reference_chosen_logps": -419.39898681640625, | |
| "debug/reference_rejected_logps": -444.2556457519531, | |
| "epoch": 0.3375, | |
| "grad_norm": 5.503733763414167, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.838258743286133, | |
| "logits/rejected": 27.663278579711914, | |
| "logps/chosen": -396.57708740234375, | |
| "logps/rejected": -454.4193115234375, | |
| "loss": 0.3097, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22821906208992004, | |
| "rewards/margins": 0.32985571026802063, | |
| "rewards/rejected": -0.10163664817810059, | |
| "step": 27 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.494264602661133, | |
| "debug/policy_chosen_logps": -386.3175048828125, | |
| "debug/policy_rejected_logits": 25.22478675842285, | |
| "debug/policy_rejected_logps": -454.5428466796875, | |
| "debug/reference_chosen_logps": -409.989501953125, | |
| "debug/reference_rejected_logps": -440.07342529296875, | |
| "epoch": 0.35, | |
| "grad_norm": 5.4619274191461775, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.494264602661133, | |
| "logits/rejected": 25.22478675842285, | |
| "logps/chosen": -386.3175048828125, | |
| "logps/rejected": -454.5428466796875, | |
| "loss": 0.3427, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.23671993613243103, | |
| "rewards/margins": 0.38141441345214844, | |
| "rewards/rejected": -0.1446944773197174, | |
| "step": 28 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.47563362121582, | |
| "debug/policy_chosen_logps": -411.8681640625, | |
| "debug/policy_rejected_logits": 29.555112838745117, | |
| "debug/policy_rejected_logps": -495.034423828125, | |
| "debug/reference_chosen_logps": -428.219970703125, | |
| "debug/reference_rejected_logps": -462.74237060546875, | |
| "epoch": 0.3625, | |
| "grad_norm": 5.291532814248025, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.47563362121582, | |
| "logits/rejected": 29.555112838745117, | |
| "logps/chosen": -411.8681640625, | |
| "logps/rejected": -495.034423828125, | |
| "loss": 0.3448, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16351768374443054, | |
| "rewards/margins": 0.486438125371933, | |
| "rewards/rejected": -0.32292044162750244, | |
| "step": 29 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.484766006469727, | |
| "debug/policy_chosen_logps": -403.1236572265625, | |
| "debug/policy_rejected_logits": 28.859256744384766, | |
| "debug/policy_rejected_logps": -473.91632080078125, | |
| "debug/reference_chosen_logps": -417.843505859375, | |
| "debug/reference_rejected_logps": -439.25738525390625, | |
| "epoch": 0.375, | |
| "grad_norm": 6.796682505768599, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.484766006469727, | |
| "logits/rejected": 28.859256744384766, | |
| "logps/chosen": -403.1236572265625, | |
| "logps/rejected": -473.91632080078125, | |
| "loss": 0.2546, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.14719845354557037, | |
| "rewards/margins": 0.4937880337238312, | |
| "rewards/rejected": -0.3465895652770996, | |
| "step": 30 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 22.867298126220703, | |
| "debug/policy_chosen_logps": -418.6160888671875, | |
| "debug/policy_rejected_logits": 28.045475006103516, | |
| "debug/policy_rejected_logps": -446.5901184082031, | |
| "debug/reference_chosen_logps": -429.67620849609375, | |
| "debug/reference_rejected_logps": -433.1312255859375, | |
| "epoch": 0.3875, | |
| "grad_norm": 7.032111537325665, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 22.867298126220703, | |
| "logits/rejected": 28.045475006103516, | |
| "logps/chosen": -418.6160888671875, | |
| "logps/rejected": -446.5901184082031, | |
| "loss": 0.3774, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.11060100793838501, | |
| "rewards/margins": 0.24519018828868866, | |
| "rewards/rejected": -0.13458918035030365, | |
| "step": 31 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 21.08600425720215, | |
| "debug/policy_chosen_logps": -425.2349853515625, | |
| "debug/policy_rejected_logits": 28.975238800048828, | |
| "debug/policy_rejected_logps": -470.2121887207031, | |
| "debug/reference_chosen_logps": -433.0067443847656, | |
| "debug/reference_rejected_logps": -453.880859375, | |
| "epoch": 0.4, | |
| "grad_norm": 6.839829841567521, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 21.08600425720215, | |
| "logits/rejected": 28.975238800048828, | |
| "logps/chosen": -425.2349853515625, | |
| "logps/rejected": -470.2121887207031, | |
| "loss": 0.2944, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.07771759480237961, | |
| "rewards/margins": 0.2410309612751007, | |
| "rewards/rejected": -0.1633133739233017, | |
| "step": 32 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 25.196168899536133, | |
| "debug/policy_chosen_logps": -411.496337890625, | |
| "debug/policy_rejected_logits": 26.846052169799805, | |
| "debug/policy_rejected_logps": -488.6064453125, | |
| "debug/reference_chosen_logps": -432.0963439941406, | |
| "debug/reference_rejected_logps": -467.6219787597656, | |
| "epoch": 0.4125, | |
| "grad_norm": 4.649169588980754, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 25.196168899536133, | |
| "logits/rejected": 26.846052169799805, | |
| "logps/chosen": -411.496337890625, | |
| "logps/rejected": -488.6064453125, | |
| "loss": 0.2653, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.20599989593029022, | |
| "rewards/margins": 0.41584473848342896, | |
| "rewards/rejected": -0.20984479784965515, | |
| "step": 33 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.207822799682617, | |
| "debug/policy_chosen_logps": -383.0249328613281, | |
| "debug/policy_rejected_logits": 32.11717224121094, | |
| "debug/policy_rejected_logps": -473.04638671875, | |
| "debug/reference_chosen_logps": -408.9935302734375, | |
| "debug/reference_rejected_logps": -424.98388671875, | |
| "epoch": 0.425, | |
| "grad_norm": 5.057572035835091, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.207822799682617, | |
| "logits/rejected": 32.11717224121094, | |
| "logps/chosen": -383.0249328613281, | |
| "logps/rejected": -473.04638671875, | |
| "loss": 0.3145, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2596858739852905, | |
| "rewards/margins": 0.740310549736023, | |
| "rewards/rejected": -0.4806246757507324, | |
| "step": 34 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.943693161010742, | |
| "debug/policy_chosen_logps": -399.00933837890625, | |
| "debug/policy_rejected_logits": 30.121450424194336, | |
| "debug/policy_rejected_logps": -462.94781494140625, | |
| "debug/reference_chosen_logps": -414.2557067871094, | |
| "debug/reference_rejected_logps": -443.25909423828125, | |
| "epoch": 0.4375, | |
| "grad_norm": 3.7317792693885603, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.943693161010742, | |
| "logits/rejected": 30.121450424194336, | |
| "logps/chosen": -399.00933837890625, | |
| "logps/rejected": -462.94781494140625, | |
| "loss": 0.2713, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.15246371924877167, | |
| "rewards/margins": 0.34935063123703003, | |
| "rewards/rejected": -0.19688692688941956, | |
| "step": 35 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.31939125061035, | |
| "debug/policy_chosen_logps": -400.14715576171875, | |
| "debug/policy_rejected_logits": 31.468852996826172, | |
| "debug/policy_rejected_logps": -513.3980712890625, | |
| "debug/reference_chosen_logps": -426.7197570800781, | |
| "debug/reference_rejected_logps": -475.2760925292969, | |
| "epoch": 0.45, | |
| "grad_norm": 4.146366903828521, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.31939125061035, | |
| "logits/rejected": 31.468852996826172, | |
| "logps/chosen": -400.14715576171875, | |
| "logps/rejected": -513.3980712890625, | |
| "loss": 0.284, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2657262682914734, | |
| "rewards/margins": 0.646946370601654, | |
| "rewards/rejected": -0.3812200725078583, | |
| "step": 36 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.51481056213379, | |
| "debug/policy_chosen_logps": -385.47406005859375, | |
| "debug/policy_rejected_logits": 30.034343719482422, | |
| "debug/policy_rejected_logps": -452.683837890625, | |
| "debug/reference_chosen_logps": -420.1163635253906, | |
| "debug/reference_rejected_logps": -444.416259765625, | |
| "epoch": 0.4625, | |
| "grad_norm": 4.656579513796896, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.51481056213379, | |
| "logits/rejected": 30.034343719482422, | |
| "logps/chosen": -385.47406005859375, | |
| "logps/rejected": -452.683837890625, | |
| "loss": 0.2506, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.34642308950424194, | |
| "rewards/margins": 0.4290991425514221, | |
| "rewards/rejected": -0.08267608284950256, | |
| "step": 37 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.45956039428711, | |
| "debug/policy_chosen_logps": -408.93182373046875, | |
| "debug/policy_rejected_logits": 26.489351272583008, | |
| "debug/policy_rejected_logps": -452.5537414550781, | |
| "debug/reference_chosen_logps": -432.7354736328125, | |
| "debug/reference_rejected_logps": -445.1405944824219, | |
| "epoch": 0.475, | |
| "grad_norm": 6.017458719962861, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.45956039428711, | |
| "logits/rejected": 26.489351272583008, | |
| "logps/chosen": -408.93182373046875, | |
| "logps/rejected": -452.5537414550781, | |
| "loss": 0.2954, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.23803669214248657, | |
| "rewards/margins": 0.3121681809425354, | |
| "rewards/rejected": -0.07413151115179062, | |
| "step": 38 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.04475975036621, | |
| "debug/policy_chosen_logps": -381.45330810546875, | |
| "debug/policy_rejected_logits": 31.125469207763672, | |
| "debug/policy_rejected_logps": -458.4730529785156, | |
| "debug/reference_chosen_logps": -411.2501525878906, | |
| "debug/reference_rejected_logps": -440.47845458984375, | |
| "epoch": 0.4875, | |
| "grad_norm": 7.182310738360888, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.04475975036621, | |
| "logits/rejected": 31.125469207763672, | |
| "logps/chosen": -381.45330810546875, | |
| "logps/rejected": -458.4730529785156, | |
| "loss": 0.2881, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.29796817898750305, | |
| "rewards/margins": 0.4779142439365387, | |
| "rewards/rejected": -0.17994609475135803, | |
| "step": 39 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.85544776916504, | |
| "debug/policy_chosen_logps": -395.41802978515625, | |
| "debug/policy_rejected_logits": 23.536022186279297, | |
| "debug/policy_rejected_logps": -410.3462829589844, | |
| "debug/reference_chosen_logps": -413.8913269042969, | |
| "debug/reference_rejected_logps": -419.77679443359375, | |
| "epoch": 0.5, | |
| "grad_norm": 7.078963246488324, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.85544776916504, | |
| "logits/rejected": 23.536022186279297, | |
| "logps/chosen": -395.41802978515625, | |
| "logps/rejected": -410.3462829589844, | |
| "loss": 0.3245, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.18473277986049652, | |
| "rewards/margins": 0.09042750298976898, | |
| "rewards/rejected": 0.09430526942014694, | |
| "step": 40 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.017606735229492, | |
| "debug/policy_chosen_logps": -385.16949462890625, | |
| "debug/policy_rejected_logits": 26.136432647705078, | |
| "debug/policy_rejected_logps": -455.77178955078125, | |
| "debug/reference_chosen_logps": -409.3904724121094, | |
| "debug/reference_rejected_logps": -446.24786376953125, | |
| "epoch": 0.5125, | |
| "grad_norm": 4.348585522756699, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.017606735229492, | |
| "logits/rejected": 26.136432647705078, | |
| "logps/chosen": -385.16949462890625, | |
| "logps/rejected": -455.77178955078125, | |
| "loss": 0.3493, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.24220973253250122, | |
| "rewards/margins": 0.33744922280311584, | |
| "rewards/rejected": -0.09523948282003403, | |
| "step": 41 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.70126724243164, | |
| "debug/policy_chosen_logps": -393.8923645019531, | |
| "debug/policy_rejected_logits": 28.3759765625, | |
| "debug/policy_rejected_logps": -474.95550537109375, | |
| "debug/reference_chosen_logps": -413.5538330078125, | |
| "debug/reference_rejected_logps": -451.510498046875, | |
| "epoch": 0.525, | |
| "grad_norm": 6.403780077024945, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.70126724243164, | |
| "logits/rejected": 28.3759765625, | |
| "logps/chosen": -393.8923645019531, | |
| "logps/rejected": -474.95550537109375, | |
| "loss": 0.2725, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.19661453366279602, | |
| "rewards/margins": 0.4310648441314697, | |
| "rewards/rejected": -0.2344503253698349, | |
| "step": 42 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.40342903137207, | |
| "debug/policy_chosen_logps": -410.5350341796875, | |
| "debug/policy_rejected_logits": 31.70136833190918, | |
| "debug/policy_rejected_logps": -484.47381591796875, | |
| "debug/reference_chosen_logps": -424.74542236328125, | |
| "debug/reference_rejected_logps": -446.88623046875, | |
| "epoch": 0.5375, | |
| "grad_norm": 9.04429983740781, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.40342903137207, | |
| "logits/rejected": 31.70136833190918, | |
| "logps/chosen": -410.5350341796875, | |
| "logps/rejected": -484.47381591796875, | |
| "loss": 0.2588, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.14210368692874908, | |
| "rewards/margins": 0.5179791450500488, | |
| "rewards/rejected": -0.37587541341781616, | |
| "step": 43 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.714599609375, | |
| "debug/policy_chosen_logps": -411.0771484375, | |
| "debug/policy_rejected_logits": 22.63053321838379, | |
| "debug/policy_rejected_logps": -460.9835510253906, | |
| "debug/reference_chosen_logps": -425.8843994140625, | |
| "debug/reference_rejected_logps": -441.40985107421875, | |
| "epoch": 0.55, | |
| "grad_norm": 5.462116354601322, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.714599609375, | |
| "logits/rejected": 22.63053321838379, | |
| "logps/chosen": -411.0771484375, | |
| "logps/rejected": -460.9835510253906, | |
| "loss": 0.3296, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.14807261526584625, | |
| "rewards/margins": 0.34380990266799927, | |
| "rewards/rejected": -0.19573725759983063, | |
| "step": 44 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 23.49720001220703, | |
| "debug/policy_chosen_logps": -402.09844970703125, | |
| "debug/policy_rejected_logits": 29.42737579345703, | |
| "debug/policy_rejected_logps": -443.2966003417969, | |
| "debug/reference_chosen_logps": -417.8492431640625, | |
| "debug/reference_rejected_logps": -440.06658935546875, | |
| "epoch": 0.5625, | |
| "grad_norm": 5.887141950804354, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 23.49720001220703, | |
| "logits/rejected": 29.42737579345703, | |
| "logps/chosen": -402.09844970703125, | |
| "logps/rejected": -443.2966003417969, | |
| "loss": 0.3101, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.15750765800476074, | |
| "rewards/margins": 0.1898074597120285, | |
| "rewards/rejected": -0.03229980170726776, | |
| "step": 45 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.707294464111328, | |
| "debug/policy_chosen_logps": -381.8944396972656, | |
| "debug/policy_rejected_logits": 26.21320343017578, | |
| "debug/policy_rejected_logps": -449.5614318847656, | |
| "debug/reference_chosen_logps": -402.29058837890625, | |
| "debug/reference_rejected_logps": -439.96856689453125, | |
| "epoch": 0.575, | |
| "grad_norm": 7.109794252989897, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.707294464111328, | |
| "logits/rejected": 26.21320343017578, | |
| "logps/chosen": -381.8944396972656, | |
| "logps/rejected": -449.5614318847656, | |
| "loss": 0.2665, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.20396174490451813, | |
| "rewards/margins": 0.2998904585838318, | |
| "rewards/rejected": -0.09592871367931366, | |
| "step": 46 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 25.76456069946289, | |
| "debug/policy_chosen_logps": -413.05450439453125, | |
| "debug/policy_rejected_logits": 29.15418815612793, | |
| "debug/policy_rejected_logps": -446.2088623046875, | |
| "debug/reference_chosen_logps": -437.90911865234375, | |
| "debug/reference_rejected_logps": -437.53997802734375, | |
| "epoch": 0.5875, | |
| "grad_norm": 4.107110423899632, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 25.76456069946289, | |
| "logits/rejected": 29.15418815612793, | |
| "logps/chosen": -413.05450439453125, | |
| "logps/rejected": -446.2088623046875, | |
| "loss": 0.2756, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.24854573607444763, | |
| "rewards/margins": 0.3352348506450653, | |
| "rewards/rejected": -0.08668914437294006, | |
| "step": 47 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.897085189819336, | |
| "debug/policy_chosen_logps": -395.95501708984375, | |
| "debug/policy_rejected_logits": 24.991113662719727, | |
| "debug/policy_rejected_logps": -443.54205322265625, | |
| "debug/reference_chosen_logps": -417.3414611816406, | |
| "debug/reference_rejected_logps": -438.97320556640625, | |
| "epoch": 0.6, | |
| "grad_norm": 3.7538316619274052, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.897085189819336, | |
| "logits/rejected": 24.991113662719727, | |
| "logps/chosen": -395.95501708984375, | |
| "logps/rejected": -443.54205322265625, | |
| "loss": 0.3109, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.21386432647705078, | |
| "rewards/margins": 0.25955283641815186, | |
| "rewards/rejected": -0.04568850249052048, | |
| "step": 48 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 17.76668357849121, | |
| "debug/policy_chosen_logps": -381.77435302734375, | |
| "debug/policy_rejected_logits": 28.45110511779785, | |
| "debug/policy_rejected_logps": -451.37335205078125, | |
| "debug/reference_chosen_logps": -409.0759582519531, | |
| "debug/reference_rejected_logps": -429.466552734375, | |
| "epoch": 0.6125, | |
| "grad_norm": 4.0217662693111516, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 17.76668357849121, | |
| "logits/rejected": 28.45110511779785, | |
| "logps/chosen": -381.77435302734375, | |
| "logps/rejected": -451.37335205078125, | |
| "loss": 0.2499, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.27301597595214844, | |
| "rewards/margins": 0.49208390712738037, | |
| "rewards/rejected": -0.21906791627407074, | |
| "step": 49 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 23.378732681274414, | |
| "debug/policy_chosen_logps": -419.51385498046875, | |
| "debug/policy_rejected_logits": 25.671220779418945, | |
| "debug/policy_rejected_logps": -420.00140380859375, | |
| "debug/reference_chosen_logps": -430.8975524902344, | |
| "debug/reference_rejected_logps": -418.96746826171875, | |
| "epoch": 0.625, | |
| "grad_norm": 4.909954105878226, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 23.378732681274414, | |
| "logits/rejected": 25.671220779418945, | |
| "logps/chosen": -419.51385498046875, | |
| "logps/rejected": -420.00140380859375, | |
| "loss": 0.315, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.11383689939975739, | |
| "rewards/margins": 0.12417624145746231, | |
| "rewards/rejected": -0.010339349508285522, | |
| "step": 50 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 23.499061584472656, | |
| "debug/policy_chosen_logps": -395.154541015625, | |
| "debug/policy_rejected_logits": 26.088699340820312, | |
| "debug/policy_rejected_logps": -437.3359375, | |
| "debug/reference_chosen_logps": -418.2652587890625, | |
| "debug/reference_rejected_logps": -433.26739501953125, | |
| "epoch": 0.6375, | |
| "grad_norm": 5.626401478627072, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 23.499061584472656, | |
| "logits/rejected": 26.088699340820312, | |
| "logps/chosen": -395.154541015625, | |
| "logps/rejected": -437.3359375, | |
| "loss": 0.2901, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.23110699653625488, | |
| "rewards/margins": 0.27179256081581116, | |
| "rewards/rejected": -0.04068557173013687, | |
| "step": 51 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 21.639236450195312, | |
| "debug/policy_chosen_logps": -402.419189453125, | |
| "debug/policy_rejected_logits": 27.019371032714844, | |
| "debug/policy_rejected_logps": -426.9820251464844, | |
| "debug/reference_chosen_logps": -423.36669921875, | |
| "debug/reference_rejected_logps": -438.14837646484375, | |
| "epoch": 0.65, | |
| "grad_norm": 5.769823478560454, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 21.639236450195312, | |
| "logits/rejected": 27.019371032714844, | |
| "logps/chosen": -402.419189453125, | |
| "logps/rejected": -426.9820251464844, | |
| "loss": 0.2988, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.20947498083114624, | |
| "rewards/margins": 0.09781143069267273, | |
| "rewards/rejected": 0.11166355758905411, | |
| "step": 52 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 23.30719757080078, | |
| "debug/policy_chosen_logps": -396.96221923828125, | |
| "debug/policy_rejected_logits": 32.520904541015625, | |
| "debug/policy_rejected_logps": -487.1607666015625, | |
| "debug/reference_chosen_logps": -425.7676696777344, | |
| "debug/reference_rejected_logps": -464.8005676269531, | |
| "epoch": 0.6625, | |
| "grad_norm": 3.958996117827781, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 23.30719757080078, | |
| "logits/rejected": 32.520904541015625, | |
| "logps/chosen": -396.96221923828125, | |
| "logps/rejected": -487.1607666015625, | |
| "loss": 0.2418, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2880541682243347, | |
| "rewards/margins": 0.5116561651229858, | |
| "rewards/rejected": -0.2236020267009735, | |
| "step": 53 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.69259262084961, | |
| "debug/policy_chosen_logps": -380.706298828125, | |
| "debug/policy_rejected_logits": 34.78654098510742, | |
| "debug/policy_rejected_logps": -464.5356140136719, | |
| "debug/reference_chosen_logps": -408.298583984375, | |
| "debug/reference_rejected_logps": -437.8012390136719, | |
| "epoch": 0.675, | |
| "grad_norm": 5.599167260869624, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.69259262084961, | |
| "logits/rejected": 34.78654098510742, | |
| "logps/chosen": -380.706298828125, | |
| "logps/rejected": -464.5356140136719, | |
| "loss": 0.2732, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.27592289447784424, | |
| "rewards/margins": 0.5432666540145874, | |
| "rewards/rejected": -0.26734381914138794, | |
| "step": 54 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.60610580444336, | |
| "debug/policy_chosen_logps": -388.4259948730469, | |
| "debug/policy_rejected_logits": 28.09248161315918, | |
| "debug/policy_rejected_logps": -442.44500732421875, | |
| "debug/reference_chosen_logps": -414.5915832519531, | |
| "debug/reference_rejected_logps": -430.89666748046875, | |
| "epoch": 0.6875, | |
| "grad_norm": 3.7882949756257345, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.60610580444336, | |
| "logits/rejected": 28.09248161315918, | |
| "logps/chosen": -388.4259948730469, | |
| "logps/rejected": -442.44500732421875, | |
| "loss": 0.3028, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2616558372974396, | |
| "rewards/margins": 0.3771390914916992, | |
| "rewards/rejected": -0.11548325419425964, | |
| "step": 55 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.7696533203125, | |
| "debug/policy_chosen_logps": -400.4974365234375, | |
| "debug/policy_rejected_logits": 30.27635383605957, | |
| "debug/policy_rejected_logps": -492.6361999511719, | |
| "debug/reference_chosen_logps": -423.21453857421875, | |
| "debug/reference_rejected_logps": -456.6308288574219, | |
| "epoch": 0.7, | |
| "grad_norm": 3.5927401238775176, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.7696533203125, | |
| "logits/rejected": 30.27635383605957, | |
| "logps/chosen": -400.4974365234375, | |
| "logps/rejected": -492.6361999511719, | |
| "loss": 0.2819, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.22717097401618958, | |
| "rewards/margins": 0.5872249007225037, | |
| "rewards/rejected": -0.3600539267063141, | |
| "step": 56 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.955514907836914, | |
| "debug/policy_chosen_logps": -407.9693603515625, | |
| "debug/policy_rejected_logits": 29.411741256713867, | |
| "debug/policy_rejected_logps": -474.837646484375, | |
| "debug/reference_chosen_logps": -418.5709228515625, | |
| "debug/reference_rejected_logps": -455.1912841796875, | |
| "epoch": 0.7125, | |
| "grad_norm": 5.251957841550457, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.955514907836914, | |
| "logits/rejected": 29.411741256713867, | |
| "logps/chosen": -407.9693603515625, | |
| "logps/rejected": -474.837646484375, | |
| "loss": 0.2251, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.10601577907800674, | |
| "rewards/margins": 0.3024793565273285, | |
| "rewards/rejected": -0.19646358489990234, | |
| "step": 57 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.137195587158203, | |
| "debug/policy_chosen_logps": -394.4432678222656, | |
| "debug/policy_rejected_logits": 25.144182205200195, | |
| "debug/policy_rejected_logps": -469.2935791015625, | |
| "debug/reference_chosen_logps": -414.51580810546875, | |
| "debug/reference_rejected_logps": -443.5001220703125, | |
| "epoch": 0.725, | |
| "grad_norm": 4.675802502460992, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.137195587158203, | |
| "logits/rejected": 25.144182205200195, | |
| "logps/chosen": -394.4432678222656, | |
| "logps/rejected": -469.2935791015625, | |
| "loss": 0.2485, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.20072558522224426, | |
| "rewards/margins": 0.4586600065231323, | |
| "rewards/rejected": -0.25793442130088806, | |
| "step": 58 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.265625, | |
| "debug/policy_chosen_logps": -439.6889953613281, | |
| "debug/policy_rejected_logits": 26.68956756591797, | |
| "debug/policy_rejected_logps": -480.45562744140625, | |
| "debug/reference_chosen_logps": -452.0250244140625, | |
| "debug/reference_rejected_logps": -461.8011474609375, | |
| "epoch": 0.7375, | |
| "grad_norm": 4.874826559571265, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.265625, | |
| "logits/rejected": 26.68956756591797, | |
| "logps/chosen": -439.6889953613281, | |
| "logps/rejected": -480.45562744140625, | |
| "loss": 0.3103, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.12336035817861557, | |
| "rewards/margins": 0.30990517139434814, | |
| "rewards/rejected": -0.18654480576515198, | |
| "step": 59 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 17.890281677246094, | |
| "debug/policy_chosen_logps": -388.18328857421875, | |
| "debug/policy_rejected_logits": 29.060598373413086, | |
| "debug/policy_rejected_logps": -480.3656921386719, | |
| "debug/reference_chosen_logps": -411.6337890625, | |
| "debug/reference_rejected_logps": -446.0444641113281, | |
| "epoch": 0.75, | |
| "grad_norm": 3.8092727948167764, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 17.890281677246094, | |
| "logits/rejected": 29.060598373413086, | |
| "logps/chosen": -388.18328857421875, | |
| "logps/rejected": -480.3656921386719, | |
| "loss": 0.3023, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.23450516164302826, | |
| "rewards/margins": 0.5777173638343811, | |
| "rewards/rejected": -0.3432122468948364, | |
| "step": 60 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 21.306364059448242, | |
| "debug/policy_chosen_logps": -417.54681396484375, | |
| "debug/policy_rejected_logits": 23.916120529174805, | |
| "debug/policy_rejected_logps": -446.38153076171875, | |
| "debug/reference_chosen_logps": -437.6956787109375, | |
| "debug/reference_rejected_logps": -429.464599609375, | |
| "epoch": 0.7625, | |
| "grad_norm": 3.732343889219544, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 21.306364059448242, | |
| "logits/rejected": 23.916120529174805, | |
| "logps/chosen": -417.54681396484375, | |
| "logps/rejected": -446.38153076171875, | |
| "loss": 0.2784, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.2014881819486618, | |
| "rewards/margins": 0.3706575632095337, | |
| "rewards/rejected": -0.16916939616203308, | |
| "step": 61 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.258764266967773, | |
| "debug/policy_chosen_logps": -400.505126953125, | |
| "debug/policy_rejected_logits": 29.247554779052734, | |
| "debug/policy_rejected_logps": -488.6958923339844, | |
| "debug/reference_chosen_logps": -424.3946533203125, | |
| "debug/reference_rejected_logps": -456.62408447265625, | |
| "epoch": 0.775, | |
| "grad_norm": 3.9288590692913727, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.258764266967773, | |
| "logits/rejected": 29.247554779052734, | |
| "logps/chosen": -400.505126953125, | |
| "logps/rejected": -488.6958923339844, | |
| "loss": 0.2174, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.23889553546905518, | |
| "rewards/margins": 0.5596135854721069, | |
| "rewards/rejected": -0.32071805000305176, | |
| "step": 62 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 25.917665481567383, | |
| "debug/policy_chosen_logps": -414.66156005859375, | |
| "debug/policy_rejected_logits": 27.687381744384766, | |
| "debug/policy_rejected_logps": -445.0428771972656, | |
| "debug/reference_chosen_logps": -433.19635009765625, | |
| "debug/reference_rejected_logps": -456.6318664550781, | |
| "epoch": 0.7875, | |
| "grad_norm": 5.374299437309603, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 25.917665481567383, | |
| "logits/rejected": 27.687381744384766, | |
| "logps/chosen": -414.66156005859375, | |
| "logps/rejected": -445.0428771972656, | |
| "loss": 0.3391, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.18534794449806213, | |
| "rewards/margins": 0.06945788860321045, | |
| "rewards/rejected": 0.11589004844427109, | |
| "step": 63 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 23.454923629760742, | |
| "debug/policy_chosen_logps": -392.32305908203125, | |
| "debug/policy_rejected_logits": 28.59387969970703, | |
| "debug/policy_rejected_logps": -434.9425048828125, | |
| "debug/reference_chosen_logps": -421.68670654296875, | |
| "debug/reference_rejected_logps": -419.7337646484375, | |
| "epoch": 0.8, | |
| "grad_norm": 6.44340155168501, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 23.454923629760742, | |
| "logits/rejected": 28.59387969970703, | |
| "logps/chosen": -392.32305908203125, | |
| "logps/rejected": -434.9425048828125, | |
| "loss": 0.286, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.29363659024238586, | |
| "rewards/margins": 0.44572439789772034, | |
| "rewards/rejected": -0.15208780765533447, | |
| "step": 64 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 24.071760177612305, | |
| "debug/policy_chosen_logps": -397.78515625, | |
| "debug/policy_rejected_logits": 24.690345764160156, | |
| "debug/policy_rejected_logps": -474.0267028808594, | |
| "debug/reference_chosen_logps": -413.3990783691406, | |
| "debug/reference_rejected_logps": -447.173095703125, | |
| "epoch": 0.8125, | |
| "grad_norm": 4.265641795749149, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 24.071760177612305, | |
| "logits/rejected": 24.690345764160156, | |
| "logps/chosen": -397.78515625, | |
| "logps/rejected": -474.0267028808594, | |
| "loss": 0.2996, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.15613901615142822, | |
| "rewards/margins": 0.42467522621154785, | |
| "rewards/rejected": -0.26853621006011963, | |
| "step": 65 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 27.323055267333984, | |
| "debug/policy_chosen_logps": -423.77105712890625, | |
| "debug/policy_rejected_logits": 32.6217041015625, | |
| "debug/policy_rejected_logps": -454.37091064453125, | |
| "debug/reference_chosen_logps": -444.37615966796875, | |
| "debug/reference_rejected_logps": -440.6434326171875, | |
| "epoch": 0.825, | |
| "grad_norm": 4.106006750125632, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 27.323055267333984, | |
| "logits/rejected": 32.6217041015625, | |
| "logps/chosen": -423.77105712890625, | |
| "logps/rejected": -454.37091064453125, | |
| "loss": 0.3016, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2060512900352478, | |
| "rewards/margins": 0.3433259427547455, | |
| "rewards/rejected": -0.13727465271949768, | |
| "step": 66 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.215694427490234, | |
| "debug/policy_chosen_logps": -392.8240966796875, | |
| "debug/policy_rejected_logits": 27.513879776000977, | |
| "debug/policy_rejected_logps": -464.026611328125, | |
| "debug/reference_chosen_logps": -423.304931640625, | |
| "debug/reference_rejected_logps": -444.7491455078125, | |
| "epoch": 0.8375, | |
| "grad_norm": 3.4052279815687894, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.215694427490234, | |
| "logits/rejected": 27.513879776000977, | |
| "logps/chosen": -392.8240966796875, | |
| "logps/rejected": -464.026611328125, | |
| "loss": 0.2463, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3048083484172821, | |
| "rewards/margins": 0.49758297204971313, | |
| "rewards/rejected": -0.19277465343475342, | |
| "step": 67 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.277034759521484, | |
| "debug/policy_chosen_logps": -390.63189697265625, | |
| "debug/policy_rejected_logits": 26.955398559570312, | |
| "debug/policy_rejected_logps": -494.57330322265625, | |
| "debug/reference_chosen_logps": -411.6377868652344, | |
| "debug/reference_rejected_logps": -470.29150390625, | |
| "epoch": 0.85, | |
| "grad_norm": 4.314646861788122, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.277034759521484, | |
| "logits/rejected": 26.955398559570312, | |
| "logps/chosen": -390.63189697265625, | |
| "logps/rejected": -494.57330322265625, | |
| "loss": 0.2748, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.21005921065807343, | |
| "rewards/margins": 0.4528772830963135, | |
| "rewards/rejected": -0.24281807243824005, | |
| "step": 68 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 17.920215606689453, | |
| "debug/policy_chosen_logps": -383.5913391113281, | |
| "debug/policy_rejected_logits": 22.582468032836914, | |
| "debug/policy_rejected_logps": -483.46142578125, | |
| "debug/reference_chosen_logps": -406.38525390625, | |
| "debug/reference_rejected_logps": -442.0745544433594, | |
| "epoch": 0.8625, | |
| "grad_norm": 7.009734793629414, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 17.920215606689453, | |
| "logits/rejected": 22.582468032836914, | |
| "logps/chosen": -383.5913391113281, | |
| "logps/rejected": -483.46142578125, | |
| "loss": 0.2491, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22793909907341003, | |
| "rewards/margins": 0.641808032989502, | |
| "rewards/rejected": -0.4138689637184143, | |
| "step": 69 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 21.91058349609375, | |
| "debug/policy_chosen_logps": -391.1068115234375, | |
| "debug/policy_rejected_logits": 28.602935791015625, | |
| "debug/policy_rejected_logps": -462.3726501464844, | |
| "debug/reference_chosen_logps": -408.93109130859375, | |
| "debug/reference_rejected_logps": -436.0927429199219, | |
| "epoch": 0.875, | |
| "grad_norm": 6.986157067049592, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 21.91058349609375, | |
| "logits/rejected": 28.602935791015625, | |
| "logps/chosen": -391.1068115234375, | |
| "logps/rejected": -462.3726501464844, | |
| "loss": 0.2593, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.1782427579164505, | |
| "rewards/margins": 0.4410419762134552, | |
| "rewards/rejected": -0.2627992033958435, | |
| "step": 70 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 19.776662826538086, | |
| "debug/policy_chosen_logps": -398.9197082519531, | |
| "debug/policy_rejected_logits": 32.09967803955078, | |
| "debug/policy_rejected_logps": -487.4455871582031, | |
| "debug/reference_chosen_logps": -420.3005676269531, | |
| "debug/reference_rejected_logps": -452.2755126953125, | |
| "epoch": 0.8875, | |
| "grad_norm": 5.014513725137227, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 19.776662826538086, | |
| "logits/rejected": 32.09967803955078, | |
| "logps/chosen": -398.9197082519531, | |
| "logps/rejected": -487.4455871582031, | |
| "loss": 0.2544, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2138085961341858, | |
| "rewards/margins": 0.5655093193054199, | |
| "rewards/rejected": -0.3517007827758789, | |
| "step": 71 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 21.157052993774414, | |
| "debug/policy_chosen_logps": -391.25897216796875, | |
| "debug/policy_rejected_logits": 26.412094116210938, | |
| "debug/policy_rejected_logps": -490.5589294433594, | |
| "debug/reference_chosen_logps": -413.3541259765625, | |
| "debug/reference_rejected_logps": -471.00921630859375, | |
| "epoch": 0.9, | |
| "grad_norm": 5.328562975855576, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 21.157052993774414, | |
| "logits/rejected": 26.412094116210938, | |
| "logps/chosen": -391.25897216796875, | |
| "logps/rejected": -490.5589294433594, | |
| "loss": 0.2412, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.2209514081478119, | |
| "rewards/margins": 0.4164482653141022, | |
| "rewards/rejected": -0.19549685716629028, | |
| "step": 72 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 24.695026397705078, | |
| "debug/policy_chosen_logps": -397.16497802734375, | |
| "debug/policy_rejected_logits": 26.946775436401367, | |
| "debug/policy_rejected_logps": -438.91900634765625, | |
| "debug/reference_chosen_logps": -418.1241455078125, | |
| "debug/reference_rejected_logps": -434.0196228027344, | |
| "epoch": 0.9125, | |
| "grad_norm": 5.591526461654656, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 24.695026397705078, | |
| "logits/rejected": 26.946775436401367, | |
| "logps/chosen": -397.16497802734375, | |
| "logps/rejected": -438.91900634765625, | |
| "loss": 0.3228, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.20959174633026123, | |
| "rewards/margins": 0.25858569145202637, | |
| "rewards/rejected": -0.04899394512176514, | |
| "step": 73 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 22.788793563842773, | |
| "debug/policy_chosen_logps": -402.5521240234375, | |
| "debug/policy_rejected_logits": 27.343595504760742, | |
| "debug/policy_rejected_logps": -457.029052734375, | |
| "debug/reference_chosen_logps": -427.92510986328125, | |
| "debug/reference_rejected_logps": -443.33807373046875, | |
| "epoch": 0.925, | |
| "grad_norm": 4.13982614264715, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 22.788793563842773, | |
| "logits/rejected": 27.343595504760742, | |
| "logps/chosen": -402.5521240234375, | |
| "logps/rejected": -457.029052734375, | |
| "loss": 0.2539, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.2537294626235962, | |
| "rewards/margins": 0.39063963294029236, | |
| "rewards/rejected": -0.13691017031669617, | |
| "step": 74 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 17.950855255126953, | |
| "debug/policy_chosen_logps": -386.80126953125, | |
| "debug/policy_rejected_logits": 28.871490478515625, | |
| "debug/policy_rejected_logps": -489.54559326171875, | |
| "debug/reference_chosen_logps": -418.87652587890625, | |
| "debug/reference_rejected_logps": -437.8173828125, | |
| "epoch": 0.9375, | |
| "grad_norm": 3.573253258125765, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 17.950855255126953, | |
| "logits/rejected": 28.871490478515625, | |
| "logps/chosen": -386.80126953125, | |
| "logps/rejected": -489.54559326171875, | |
| "loss": 0.235, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3207521438598633, | |
| "rewards/margins": 0.8380340933799744, | |
| "rewards/rejected": -0.5172819495201111, | |
| "step": 75 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.622098922729492, | |
| "debug/policy_chosen_logps": -384.69195556640625, | |
| "debug/policy_rejected_logits": 31.19890594482422, | |
| "debug/policy_rejected_logps": -452.1475524902344, | |
| "debug/reference_chosen_logps": -407.402099609375, | |
| "debug/reference_rejected_logps": -432.25592041015625, | |
| "epoch": 0.95, | |
| "grad_norm": 6.4154538717209695, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.622098922729492, | |
| "logits/rejected": 31.19890594482422, | |
| "logps/chosen": -384.69195556640625, | |
| "logps/rejected": -452.1475524902344, | |
| "loss": 0.3166, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.22710174322128296, | |
| "rewards/margins": 0.42601796984672546, | |
| "rewards/rejected": -0.1989162415266037, | |
| "step": 76 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 21.178997039794922, | |
| "debug/policy_chosen_logps": -384.0088806152344, | |
| "debug/policy_rejected_logits": 32.716346740722656, | |
| "debug/policy_rejected_logps": -477.344482421875, | |
| "debug/reference_chosen_logps": -413.40850830078125, | |
| "debug/reference_rejected_logps": -443.79180908203125, | |
| "epoch": 0.9625, | |
| "grad_norm": 4.479488616499859, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 21.178997039794922, | |
| "logits/rejected": 32.716346740722656, | |
| "logps/chosen": -384.0088806152344, | |
| "logps/rejected": -477.344482421875, | |
| "loss": 0.2545, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.2939964532852173, | |
| "rewards/margins": 0.6295228600502014, | |
| "rewards/rejected": -0.33552637696266174, | |
| "step": 77 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 25.3104190826416, | |
| "debug/policy_chosen_logps": -409.6578063964844, | |
| "debug/policy_rejected_logits": 31.45233917236328, | |
| "debug/policy_rejected_logps": -462.6581115722656, | |
| "debug/reference_chosen_logps": -426.9599609375, | |
| "debug/reference_rejected_logps": -430.9308166503906, | |
| "epoch": 0.975, | |
| "grad_norm": 3.3409543416332235, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 25.3104190826416, | |
| "logits/rejected": 31.45233917236328, | |
| "logps/chosen": -409.6578063964844, | |
| "logps/rejected": -462.6581115722656, | |
| "loss": 0.2304, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.17302149534225464, | |
| "rewards/margins": 0.49029436707496643, | |
| "rewards/rejected": -0.3172728717327118, | |
| "step": 78 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 20.39267349243164, | |
| "debug/policy_chosen_logps": -391.23089599609375, | |
| "debug/policy_rejected_logits": 29.421716690063477, | |
| "debug/policy_rejected_logps": -489.40728759765625, | |
| "debug/reference_chosen_logps": -423.12115478515625, | |
| "debug/reference_rejected_logps": -447.7901611328125, | |
| "epoch": 0.9875, | |
| "grad_norm": 5.428664997337287, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 20.39267349243164, | |
| "logits/rejected": 29.421716690063477, | |
| "logps/chosen": -391.23089599609375, | |
| "logps/rejected": -489.40728759765625, | |
| "loss": 0.2057, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3189027011394501, | |
| "rewards/margins": 0.7350739240646362, | |
| "rewards/rejected": -0.41617119312286377, | |
| "step": 79 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 18.865188598632812, | |
| "debug/policy_chosen_logps": -384.6566162109375, | |
| "debug/policy_rejected_logits": 29.98931884765625, | |
| "debug/policy_rejected_logps": -469.95892333984375, | |
| "debug/reference_chosen_logps": -407.10784912109375, | |
| "debug/reference_rejected_logps": -433.1329345703125, | |
| "epoch": 1.0, | |
| "grad_norm": 4.751087690542253, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 18.865188598632812, | |
| "logits/rejected": 29.98931884765625, | |
| "logps/chosen": -384.6566162109375, | |
| "logps/rejected": -469.95892333984375, | |
| "loss": 0.1778, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.22451260685920715, | |
| "rewards/margins": 0.5927724838256836, | |
| "rewards/rejected": -0.36825984716415405, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 80, | |
| "total_flos": 0.0, | |
| "train_loss": 0.3231356654316187, | |
| "train_runtime": 754.2708, | |
| "train_samples_per_second": 6.788, | |
| "train_steps_per_second": 0.106 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 80, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |