| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 47, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "debug/policy_chosen_logits": 31.700279235839844, | |
| "debug/policy_chosen_logps": -434.26495361328125, | |
| "debug/policy_rejected_logits": 33.99253845214844, | |
| "debug/policy_rejected_logps": -441.9063720703125, | |
| "debug/reference_chosen_logps": -434.26495361328125, | |
| "debug/reference_rejected_logps": -441.9063720703125, | |
| "epoch": 0.02127659574468085, | |
| "grad_norm": 5.407328059506411, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 31.700279235839844, | |
| "logits/rejected": 33.99253845214844, | |
| "logps/chosen": -434.26495361328125, | |
| "logps/rejected": -441.9063720703125, | |
| "loss": 0.5, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 27.489120483398438, | |
| "debug/policy_chosen_logps": -410.28472900390625, | |
| "debug/policy_rejected_logits": 31.382970809936523, | |
| "debug/policy_rejected_logps": -435.17218017578125, | |
| "debug/reference_chosen_logps": -410.96771240234375, | |
| "debug/reference_rejected_logps": -436.0491638183594, | |
| "epoch": 0.0425531914893617, | |
| "grad_norm": 4.951759612240564, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 27.489120483398438, | |
| "logits/rejected": 31.382970809936523, | |
| "logps/chosen": -410.28472900390625, | |
| "logps/rejected": -435.17218017578125, | |
| "loss": 0.5009, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.006829871796071529, | |
| "rewards/margins": -0.001939887646585703, | |
| "rewards/rejected": 0.00876975990831852, | |
| "step": 2 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 31.120014190673828, | |
| "debug/policy_chosen_logps": -401.9127197265625, | |
| "debug/policy_rejected_logits": 33.329689025878906, | |
| "debug/policy_rejected_logps": -424.90576171875, | |
| "debug/reference_chosen_logps": -402.24658203125, | |
| "debug/reference_rejected_logps": -424.7574462890625, | |
| "epoch": 0.06382978723404255, | |
| "grad_norm": 5.411851250945231, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 31.120014190673828, | |
| "logits/rejected": 33.329689025878906, | |
| "logps/chosen": -401.9127197265625, | |
| "logps/rejected": -424.90576171875, | |
| "loss": 0.5007, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.003338394220918417, | |
| "rewards/margins": 0.004821510519832373, | |
| "rewards/rejected": -0.001483116764575243, | |
| "step": 3 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 30.066335678100586, | |
| "debug/policy_chosen_logps": -403.6931457519531, | |
| "debug/policy_rejected_logits": 33.15522384643555, | |
| "debug/policy_rejected_logps": -436.77801513671875, | |
| "debug/reference_chosen_logps": -403.52996826171875, | |
| "debug/reference_rejected_logps": -436.29296875, | |
| "epoch": 0.0851063829787234, | |
| "grad_norm": 6.440647306952527, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 30.066335678100586, | |
| "logits/rejected": 33.15522384643555, | |
| "logps/chosen": -403.6931457519531, | |
| "logps/rejected": -436.77801513671875, | |
| "loss": 0.4993, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.0016318517737090588, | |
| "rewards/margins": 0.003218421945348382, | |
| "rewards/rejected": -0.00485027302056551, | |
| "step": 4 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.028703689575195, | |
| "debug/policy_chosen_logps": -391.64715576171875, | |
| "debug/policy_rejected_logits": 30.672496795654297, | |
| "debug/policy_rejected_logps": -420.91143798828125, | |
| "debug/reference_chosen_logps": -392.4825134277344, | |
| "debug/reference_rejected_logps": -422.0054931640625, | |
| "epoch": 0.10638297872340426, | |
| "grad_norm": 5.433518269383661, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.028703689575195, | |
| "logits/rejected": 30.672496795654297, | |
| "logps/chosen": -391.64715576171875, | |
| "logps/rejected": -420.91143798828125, | |
| "loss": 0.4987, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.008353347890079021, | |
| "rewards/margins": -0.0025872797705233097, | |
| "rewards/rejected": 0.010940628126263618, | |
| "step": 5 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 26.0015811920166, | |
| "debug/policy_chosen_logps": -403.2935791015625, | |
| "debug/policy_rejected_logits": 25.2414608001709, | |
| "debug/policy_rejected_logps": -407.3106994628906, | |
| "debug/reference_chosen_logps": -402.87139892578125, | |
| "debug/reference_rejected_logps": -406.1798095703125, | |
| "epoch": 0.1276595744680851, | |
| "grad_norm": 5.048275333461177, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 26.0015811920166, | |
| "logits/rejected": 25.2414608001709, | |
| "logps/chosen": -403.2935791015625, | |
| "logps/rejected": -407.3106994628906, | |
| "loss": 0.496, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0042218780145049095, | |
| "rewards/margins": 0.007086906582117081, | |
| "rewards/rejected": -0.01130878459662199, | |
| "step": 6 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 27.019393920898438, | |
| "debug/policy_chosen_logps": -424.65643310546875, | |
| "debug/policy_rejected_logits": 29.134994506835938, | |
| "debug/policy_rejected_logps": -420.3319396972656, | |
| "debug/reference_chosen_logps": -423.5784912109375, | |
| "debug/reference_rejected_logps": -420.0720520019531, | |
| "epoch": 0.14893617021276595, | |
| "grad_norm": 6.092801689841109, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 27.019393920898438, | |
| "logits/rejected": 29.134994506835938, | |
| "logps/chosen": -424.65643310546875, | |
| "logps/rejected": -420.3319396972656, | |
| "loss": 0.499, | |
| "rewards/accuracies": 0.125, | |
| "rewards/chosen": -0.010779608972370625, | |
| "rewards/margins": -0.008180923759937286, | |
| "rewards/rejected": -0.0025986863765865564, | |
| "step": 7 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 25.41461944580078, | |
| "debug/policy_chosen_logps": -420.9305419921875, | |
| "debug/policy_rejected_logits": 25.522966384887695, | |
| "debug/policy_rejected_logps": -427.20623779296875, | |
| "debug/reference_chosen_logps": -420.0583190917969, | |
| "debug/reference_rejected_logps": -426.08453369140625, | |
| "epoch": 0.1702127659574468, | |
| "grad_norm": 5.248588154856571, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 25.41461944580078, | |
| "logits/rejected": 25.522966384887695, | |
| "logps/chosen": -420.9305419921875, | |
| "logps/rejected": -427.20623779296875, | |
| "loss": 0.4975, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.00872222799807787, | |
| "rewards/margins": 0.0024948506616055965, | |
| "rewards/rejected": -0.011217079125344753, | |
| "step": 8 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 27.28373146057129, | |
| "debug/policy_chosen_logps": -413.857177734375, | |
| "debug/policy_rejected_logits": 29.01516342163086, | |
| "debug/policy_rejected_logps": -429.70623779296875, | |
| "debug/reference_chosen_logps": -413.364501953125, | |
| "debug/reference_rejected_logps": -429.416259765625, | |
| "epoch": 0.19148936170212766, | |
| "grad_norm": 5.836663309727503, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 27.28373146057129, | |
| "logits/rejected": 29.01516342163086, | |
| "logps/chosen": -413.857177734375, | |
| "logps/rejected": -429.70623779296875, | |
| "loss": 0.4954, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.004926986526697874, | |
| "rewards/margins": -0.002027016133069992, | |
| "rewards/rejected": -0.002899970393627882, | |
| "step": 9 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.55730628967285, | |
| "debug/policy_chosen_logps": -418.648193359375, | |
| "debug/policy_rejected_logits": 30.004676818847656, | |
| "debug/policy_rejected_logps": -430.01788330078125, | |
| "debug/reference_chosen_logps": -418.4599609375, | |
| "debug/reference_rejected_logps": -429.8154296875, | |
| "epoch": 0.2127659574468085, | |
| "grad_norm": 4.978725766808406, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.55730628967285, | |
| "logits/rejected": 30.004676818847656, | |
| "logps/chosen": -418.648193359375, | |
| "logps/rejected": -430.01788330078125, | |
| "loss": 0.4991, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0018823242280632257, | |
| "rewards/margins": 0.00014217384159564972, | |
| "rewards/rejected": -0.002024497603997588, | |
| "step": 10 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 33.0296516418457, | |
| "debug/policy_chosen_logps": -434.47308349609375, | |
| "debug/policy_rejected_logits": 31.160263061523438, | |
| "debug/policy_rejected_logps": -406.6353759765625, | |
| "debug/reference_chosen_logps": -433.95892333984375, | |
| "debug/reference_rejected_logps": -405.2964782714844, | |
| "epoch": 0.23404255319148937, | |
| "grad_norm": 5.822177618834045, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 33.0296516418457, | |
| "logits/rejected": 31.160263061523438, | |
| "logps/chosen": -434.47308349609375, | |
| "logps/rejected": -406.6353759765625, | |
| "loss": 0.493, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.005142059177160263, | |
| "rewards/margins": 0.008247108198702335, | |
| "rewards/rejected": -0.013389168307185173, | |
| "step": 11 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 30.720827102661133, | |
| "debug/policy_chosen_logps": -455.06597900390625, | |
| "debug/policy_rejected_logits": 32.45933151245117, | |
| "debug/policy_rejected_logps": -462.2677307128906, | |
| "debug/reference_chosen_logps": -454.6126403808594, | |
| "debug/reference_rejected_logps": -459.7181396484375, | |
| "epoch": 0.2553191489361702, | |
| "grad_norm": 5.239137130887116, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 30.720827102661133, | |
| "logits/rejected": 32.45933151245117, | |
| "logps/chosen": -455.06597900390625, | |
| "logps/rejected": -462.2677307128906, | |
| "loss": 0.4991, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.00453338585793972, | |
| "rewards/margins": 0.020962638780474663, | |
| "rewards/rejected": -0.025496024638414383, | |
| "step": 12 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 30.186174392700195, | |
| "debug/policy_chosen_logps": -412.29742431640625, | |
| "debug/policy_rejected_logits": 28.243711471557617, | |
| "debug/policy_rejected_logps": -426.9504089355469, | |
| "debug/reference_chosen_logps": -411.92120361328125, | |
| "debug/reference_rejected_logps": -425.7698974609375, | |
| "epoch": 0.2765957446808511, | |
| "grad_norm": 5.6983956081800855, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 30.186174392700195, | |
| "logits/rejected": 28.243711471557617, | |
| "logps/chosen": -412.29742431640625, | |
| "logps/rejected": -426.9504089355469, | |
| "loss": 0.4955, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.003762359730899334, | |
| "rewards/margins": 0.008042870089411736, | |
| "rewards/rejected": -0.011805228888988495, | |
| "step": 13 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.8179931640625, | |
| "debug/policy_chosen_logps": -402.04205322265625, | |
| "debug/policy_rejected_logits": 27.887521743774414, | |
| "debug/policy_rejected_logps": -406.5090637207031, | |
| "debug/reference_chosen_logps": -402.81463623046875, | |
| "debug/reference_rejected_logps": -406.35760498046875, | |
| "epoch": 0.2978723404255319, | |
| "grad_norm": 5.185829515819964, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.8179931640625, | |
| "logits/rejected": 27.887521743774414, | |
| "logps/chosen": -402.04205322265625, | |
| "logps/rejected": -406.5090637207031, | |
| "loss": 0.4892, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.007725906558334827, | |
| "rewards/margins": 0.009240342304110527, | |
| "rewards/rejected": -0.0015144352801144123, | |
| "step": 14 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 30.058448791503906, | |
| "debug/policy_chosen_logps": -412.29827880859375, | |
| "debug/policy_rejected_logits": 29.466854095458984, | |
| "debug/policy_rejected_logps": -412.73504638671875, | |
| "debug/reference_chosen_logps": -411.7734680175781, | |
| "debug/reference_rejected_logps": -413.09912109375, | |
| "epoch": 0.3191489361702128, | |
| "grad_norm": 5.147098230721813, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 30.058448791503906, | |
| "logits/rejected": 29.466854095458984, | |
| "logps/chosen": -412.29827880859375, | |
| "logps/rejected": -412.73504638671875, | |
| "loss": 0.4893, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.005248222034424543, | |
| "rewards/margins": -0.008889121934771538, | |
| "rewards/rejected": 0.0036408999003469944, | |
| "step": 15 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 26.801280975341797, | |
| "debug/policy_chosen_logps": -453.10833740234375, | |
| "debug/policy_rejected_logits": 28.296146392822266, | |
| "debug/policy_rejected_logps": -433.0950927734375, | |
| "debug/reference_chosen_logps": -453.86102294921875, | |
| "debug/reference_rejected_logps": -432.42510986328125, | |
| "epoch": 0.3404255319148936, | |
| "grad_norm": 5.3650227932794765, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 26.801280975341797, | |
| "logits/rejected": 28.296146392822266, | |
| "logps/chosen": -453.10833740234375, | |
| "logps/rejected": -433.0950927734375, | |
| "loss": 0.4907, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.007526512257754803, | |
| "rewards/margins": 0.014225959777832031, | |
| "rewards/rejected": -0.0066994475200772285, | |
| "step": 16 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 25.992467880249023, | |
| "debug/policy_chosen_logps": -436.1377258300781, | |
| "debug/policy_rejected_logits": 27.410860061645508, | |
| "debug/policy_rejected_logps": -426.43035888671875, | |
| "debug/reference_chosen_logps": -434.6832275390625, | |
| "debug/reference_rejected_logps": -424.5072021484375, | |
| "epoch": 0.3617021276595745, | |
| "grad_norm": 5.048216336945854, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 25.992467880249023, | |
| "logits/rejected": 27.410860061645508, | |
| "logps/chosen": -436.1377258300781, | |
| "logps/rejected": -426.43035888671875, | |
| "loss": 0.494, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.014545059762895107, | |
| "rewards/margins": 0.004686659201979637, | |
| "rewards/rejected": -0.01923171989619732, | |
| "step": 17 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 27.924072265625, | |
| "debug/policy_chosen_logps": -456.6978759765625, | |
| "debug/policy_rejected_logits": 27.263843536376953, | |
| "debug/policy_rejected_logps": -411.67791748046875, | |
| "debug/reference_chosen_logps": -455.6437683105469, | |
| "debug/reference_rejected_logps": -408.3628234863281, | |
| "epoch": 0.3829787234042553, | |
| "grad_norm": 4.959644897985259, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 27.924072265625, | |
| "logits/rejected": 27.263843536376953, | |
| "logps/chosen": -456.6978759765625, | |
| "logps/rejected": -411.67791748046875, | |
| "loss": 0.4872, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.01054123044013977, | |
| "rewards/margins": 0.022609787061810493, | |
| "rewards/rejected": -0.033151015639305115, | |
| "step": 18 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 30.296974182128906, | |
| "debug/policy_chosen_logps": -407.5791320800781, | |
| "debug/policy_rejected_logits": 29.760583877563477, | |
| "debug/policy_rejected_logps": -417.291748046875, | |
| "debug/reference_chosen_logps": -410.10662841796875, | |
| "debug/reference_rejected_logps": -418.6151123046875, | |
| "epoch": 0.40425531914893614, | |
| "grad_norm": 4.809418559441442, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 30.296974182128906, | |
| "logits/rejected": 29.760583877563477, | |
| "logps/chosen": -407.5791320800781, | |
| "logps/rejected": -417.291748046875, | |
| "loss": 0.4937, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.02527473494410515, | |
| "rewards/margins": 0.012040939182043076, | |
| "rewards/rejected": 0.013233794830739498, | |
| "step": 19 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 30.575592041015625, | |
| "debug/policy_chosen_logps": -413.82574462890625, | |
| "debug/policy_rejected_logits": 32.98490905761719, | |
| "debug/policy_rejected_logps": -443.43548583984375, | |
| "debug/reference_chosen_logps": -414.27642822265625, | |
| "debug/reference_rejected_logps": -441.5928649902344, | |
| "epoch": 0.425531914893617, | |
| "grad_norm": 5.055368747694493, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 30.575592041015625, | |
| "logits/rejected": 32.98490905761719, | |
| "logps/chosen": -413.82574462890625, | |
| "logps/rejected": -443.43548583984375, | |
| "loss": 0.4771, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.0045069498009979725, | |
| "rewards/margins": 0.02293361723423004, | |
| "rewards/rejected": -0.018426666036248207, | |
| "step": 20 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.962617874145508, | |
| "debug/policy_chosen_logps": -416.78582763671875, | |
| "debug/policy_rejected_logits": 31.380332946777344, | |
| "debug/policy_rejected_logps": -443.6494445800781, | |
| "debug/reference_chosen_logps": -419.51043701171875, | |
| "debug/reference_rejected_logps": -442.1171875, | |
| "epoch": 0.44680851063829785, | |
| "grad_norm": 5.2399641694392685, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.962617874145508, | |
| "logits/rejected": 31.380332946777344, | |
| "logps/chosen": -416.78582763671875, | |
| "logps/rejected": -443.6494445800781, | |
| "loss": 0.4861, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.02724616974592209, | |
| "rewards/margins": 0.04256858676671982, | |
| "rewards/rejected": -0.015322417952120304, | |
| "step": 21 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 33.89327621459961, | |
| "debug/policy_chosen_logps": -439.533203125, | |
| "debug/policy_rejected_logits": 32.8599853515625, | |
| "debug/policy_rejected_logps": -468.5189208984375, | |
| "debug/reference_chosen_logps": -437.9319763183594, | |
| "debug/reference_rejected_logps": -460.019287109375, | |
| "epoch": 0.46808510638297873, | |
| "grad_norm": 5.600180944400873, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 33.89327621459961, | |
| "logits/rejected": 32.8599853515625, | |
| "logps/chosen": -439.533203125, | |
| "logps/rejected": -468.5189208984375, | |
| "loss": 0.4779, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.016012268140912056, | |
| "rewards/margins": 0.06898414343595505, | |
| "rewards/rejected": -0.08499641716480255, | |
| "step": 22 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 30.022546768188477, | |
| "debug/policy_chosen_logps": -448.22540283203125, | |
| "debug/policy_rejected_logits": 30.50183868408203, | |
| "debug/policy_rejected_logps": -417.8683776855469, | |
| "debug/reference_chosen_logps": -448.73858642578125, | |
| "debug/reference_rejected_logps": -417.9635925292969, | |
| "epoch": 0.48936170212765956, | |
| "grad_norm": 5.053771531276715, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 30.022546768188477, | |
| "logits/rejected": 30.50183868408203, | |
| "logps/chosen": -448.22540283203125, | |
| "logps/rejected": -417.8683776855469, | |
| "loss": 0.4906, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0051317219622433186, | |
| "rewards/margins": 0.004179535433650017, | |
| "rewards/rejected": 0.0009521869942545891, | |
| "step": 23 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 31.068572998046875, | |
| "debug/policy_chosen_logps": -408.3885192871094, | |
| "debug/policy_rejected_logits": 30.79738426208496, | |
| "debug/policy_rejected_logps": -432.73651123046875, | |
| "debug/reference_chosen_logps": -406.73419189453125, | |
| "debug/reference_rejected_logps": -432.0497131347656, | |
| "epoch": 0.5106382978723404, | |
| "grad_norm": 5.120783229464688, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 31.068572998046875, | |
| "logits/rejected": 30.79738426208496, | |
| "logps/chosen": -408.3885192871094, | |
| "logps/rejected": -432.73651123046875, | |
| "loss": 0.478, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.016543272882699966, | |
| "rewards/margins": -0.009675255045294762, | |
| "rewards/rejected": -0.006868018768727779, | |
| "step": 24 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.878725051879883, | |
| "debug/policy_chosen_logps": -434.04144287109375, | |
| "debug/policy_rejected_logits": 30.279621124267578, | |
| "debug/policy_rejected_logps": -457.3016357421875, | |
| "debug/reference_chosen_logps": -433.21746826171875, | |
| "debug/reference_rejected_logps": -453.06280517578125, | |
| "epoch": 0.5319148936170213, | |
| "grad_norm": 5.297697739276052, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.878725051879883, | |
| "logits/rejected": 30.279621124267578, | |
| "logps/chosen": -434.04144287109375, | |
| "logps/rejected": -457.3016357421875, | |
| "loss": 0.485, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.00823978427797556, | |
| "rewards/margins": 0.03414863348007202, | |
| "rewards/rejected": -0.042388420552015305, | |
| "step": 25 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 30.609947204589844, | |
| "debug/policy_chosen_logps": -402.73504638671875, | |
| "debug/policy_rejected_logits": 29.12665367126465, | |
| "debug/policy_rejected_logps": -411.1260986328125, | |
| "debug/reference_chosen_logps": -404.96392822265625, | |
| "debug/reference_rejected_logps": -411.49969482421875, | |
| "epoch": 0.5531914893617021, | |
| "grad_norm": 5.066192133102437, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 30.609947204589844, | |
| "logits/rejected": 29.12665367126465, | |
| "logps/chosen": -402.73504638671875, | |
| "logps/rejected": -411.1260986328125, | |
| "loss": 0.4781, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.022288817912340164, | |
| "rewards/margins": 0.01855243556201458, | |
| "rewards/rejected": 0.003736380487680435, | |
| "step": 26 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 26.225852966308594, | |
| "debug/policy_chosen_logps": -434.0633544921875, | |
| "debug/policy_rejected_logits": 27.547882080078125, | |
| "debug/policy_rejected_logps": -460.3682861328125, | |
| "debug/reference_chosen_logps": -434.0380859375, | |
| "debug/reference_rejected_logps": -457.2252197265625, | |
| "epoch": 0.574468085106383, | |
| "grad_norm": 5.199359660864606, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 26.225852966308594, | |
| "logits/rejected": 27.547882080078125, | |
| "logps/chosen": -434.0633544921875, | |
| "logps/rejected": -460.3682861328125, | |
| "loss": 0.4775, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.0002528773620724678, | |
| "rewards/margins": 0.031177710741758347, | |
| "rewards/rejected": -0.03143058717250824, | |
| "step": 27 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 34.810482025146484, | |
| "debug/policy_chosen_logps": -432.9176025390625, | |
| "debug/policy_rejected_logits": 32.29673385620117, | |
| "debug/policy_rejected_logps": -435.6657409667969, | |
| "debug/reference_chosen_logps": -433.37603759765625, | |
| "debug/reference_rejected_logps": -432.38958740234375, | |
| "epoch": 0.5957446808510638, | |
| "grad_norm": 5.456307945174751, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 34.810482025146484, | |
| "logits/rejected": 32.29673385620117, | |
| "logps/chosen": -432.9176025390625, | |
| "logps/rejected": -435.6657409667969, | |
| "loss": 0.4635, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.004584426060318947, | |
| "rewards/margins": 0.03734596073627472, | |
| "rewards/rejected": -0.03276153653860092, | |
| "step": 28 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.973360061645508, | |
| "debug/policy_chosen_logps": -432.0859375, | |
| "debug/policy_rejected_logits": 27.616941452026367, | |
| "debug/policy_rejected_logps": -419.5810546875, | |
| "debug/reference_chosen_logps": -432.6524658203125, | |
| "debug/reference_rejected_logps": -413.3448181152344, | |
| "epoch": 0.6170212765957447, | |
| "grad_norm": 4.889651674840413, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.973360061645508, | |
| "logits/rejected": 27.616941452026367, | |
| "logps/chosen": -432.0859375, | |
| "logps/rejected": -419.5810546875, | |
| "loss": 0.4789, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.005664861761033535, | |
| "rewards/margins": 0.06802742183208466, | |
| "rewards/rejected": -0.062362559139728546, | |
| "step": 29 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.820457458496094, | |
| "debug/policy_chosen_logps": -419.7233581542969, | |
| "debug/policy_rejected_logits": 30.256000518798828, | |
| "debug/policy_rejected_logps": -422.4107971191406, | |
| "debug/reference_chosen_logps": -423.3680725097656, | |
| "debug/reference_rejected_logps": -421.3091125488281, | |
| "epoch": 0.6382978723404256, | |
| "grad_norm": 5.137030977785722, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.820457458496094, | |
| "logits/rejected": 30.256000518798828, | |
| "logps/chosen": -419.7233581542969, | |
| "logps/rejected": -422.4107971191406, | |
| "loss": 0.4823, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.03644702956080437, | |
| "rewards/margins": 0.047463756054639816, | |
| "rewards/rejected": -0.011016730219125748, | |
| "step": 30 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.283926010131836, | |
| "debug/policy_chosen_logps": -392.496826171875, | |
| "debug/policy_rejected_logits": 31.77328109741211, | |
| "debug/policy_rejected_logps": -434.51806640625, | |
| "debug/reference_chosen_logps": -395.81146240234375, | |
| "debug/reference_rejected_logps": -434.8221435546875, | |
| "epoch": 0.6595744680851063, | |
| "grad_norm": 4.951189622094444, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.283926010131836, | |
| "logits/rejected": 31.77328109741211, | |
| "logps/chosen": -392.496826171875, | |
| "logps/rejected": -434.51806640625, | |
| "loss": 0.4638, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.033146705478429794, | |
| "rewards/margins": 0.03010578267276287, | |
| "rewards/rejected": 0.003040926530957222, | |
| "step": 31 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.353422164916992, | |
| "debug/policy_chosen_logps": -414.32415771484375, | |
| "debug/policy_rejected_logits": 30.822248458862305, | |
| "debug/policy_rejected_logps": -430.6376037597656, | |
| "debug/reference_chosen_logps": -415.54888916015625, | |
| "debug/reference_rejected_logps": -431.1400146484375, | |
| "epoch": 0.6808510638297872, | |
| "grad_norm": 4.999575032095535, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.353422164916992, | |
| "logits/rejected": 30.822248458862305, | |
| "logps/chosen": -414.32415771484375, | |
| "logps/rejected": -430.6376037597656, | |
| "loss": 0.4834, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.012247240170836449, | |
| "rewards/margins": 0.007223015185445547, | |
| "rewards/rejected": 0.00502422172576189, | |
| "step": 32 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 27.81666374206543, | |
| "debug/policy_chosen_logps": -437.5671081542969, | |
| "debug/policy_rejected_logits": 29.937236785888672, | |
| "debug/policy_rejected_logps": -429.474853515625, | |
| "debug/reference_chosen_logps": -440.85504150390625, | |
| "debug/reference_rejected_logps": -431.23309326171875, | |
| "epoch": 0.7021276595744681, | |
| "grad_norm": 5.116678809536897, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 27.81666374206543, | |
| "logits/rejected": 29.937236785888672, | |
| "logps/chosen": -437.5671081542969, | |
| "logps/rejected": -429.474853515625, | |
| "loss": 0.4783, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.03287952393293381, | |
| "rewards/margins": 0.01529712788760662, | |
| "rewards/rejected": 0.017582397907972336, | |
| "step": 33 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 32.981014251708984, | |
| "debug/policy_chosen_logps": -450.77899169921875, | |
| "debug/policy_rejected_logits": 29.245454788208008, | |
| "debug/policy_rejected_logps": -421.23468017578125, | |
| "debug/reference_chosen_logps": -454.22735595703125, | |
| "debug/reference_rejected_logps": -425.0584716796875, | |
| "epoch": 0.723404255319149, | |
| "grad_norm": 5.485713658124459, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 32.981014251708984, | |
| "logits/rejected": 29.245454788208008, | |
| "logps/chosen": -450.77899169921875, | |
| "logps/rejected": -421.23468017578125, | |
| "loss": 0.4736, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.03448398783802986, | |
| "rewards/margins": -0.003754120320081711, | |
| "rewards/rejected": 0.038238104432821274, | |
| "step": 34 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 30.9548397064209, | |
| "debug/policy_chosen_logps": -427.41632080078125, | |
| "debug/policy_rejected_logits": 29.430871963500977, | |
| "debug/policy_rejected_logps": -436.83050537109375, | |
| "debug/reference_chosen_logps": -430.6944580078125, | |
| "debug/reference_rejected_logps": -433.17041015625, | |
| "epoch": 0.7446808510638298, | |
| "grad_norm": 5.73163901812078, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 30.9548397064209, | |
| "logits/rejected": 29.430871963500977, | |
| "logps/chosen": -427.41632080078125, | |
| "logps/rejected": -436.83050537109375, | |
| "loss": 0.4459, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.03278125822544098, | |
| "rewards/margins": 0.06938225030899048, | |
| "rewards/rejected": -0.0366009883582592, | |
| "step": 35 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.33571434020996, | |
| "debug/policy_chosen_logps": -400.690673828125, | |
| "debug/policy_rejected_logits": 24.813756942749023, | |
| "debug/policy_rejected_logps": -445.54791259765625, | |
| "debug/reference_chosen_logps": -404.2062072753906, | |
| "debug/reference_rejected_logps": -433.4603576660156, | |
| "epoch": 0.7659574468085106, | |
| "grad_norm": 5.207209058021427, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.33571434020996, | |
| "logits/rejected": 24.813756942749023, | |
| "logps/chosen": -400.690673828125, | |
| "logps/rejected": -445.54791259765625, | |
| "loss": 0.4546, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.03515518084168434, | |
| "rewards/margins": 0.15603074431419373, | |
| "rewards/rejected": -0.12087554484605789, | |
| "step": 36 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.32408332824707, | |
| "debug/policy_chosen_logps": -427.9587707519531, | |
| "debug/policy_rejected_logits": 27.91067123413086, | |
| "debug/policy_rejected_logps": -405.4839172363281, | |
| "debug/reference_chosen_logps": -427.9952392578125, | |
| "debug/reference_rejected_logps": -407.0904846191406, | |
| "epoch": 0.7872340425531915, | |
| "grad_norm": 5.293688440117633, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.32408332824707, | |
| "logits/rejected": 27.91067123413086, | |
| "logps/chosen": -427.9587707519531, | |
| "logps/rejected": -405.4839172363281, | |
| "loss": 0.5106, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.00036445818841457367, | |
| "rewards/margins": -0.01570144295692444, | |
| "rewards/rejected": 0.01606590300798416, | |
| "step": 37 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.01889419555664, | |
| "debug/policy_chosen_logps": -404.5608825683594, | |
| "debug/policy_rejected_logits": 28.02815055847168, | |
| "debug/policy_rejected_logps": -413.2740173339844, | |
| "debug/reference_chosen_logps": -405.4373779296875, | |
| "debug/reference_rejected_logps": -406.2366027832031, | |
| "epoch": 0.8085106382978723, | |
| "grad_norm": 4.841695423817661, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.01889419555664, | |
| "logits/rejected": 28.02815055847168, | |
| "logps/chosen": -404.5608825683594, | |
| "logps/rejected": -413.2740173339844, | |
| "loss": 0.4719, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.00876510702073574, | |
| "rewards/margins": 0.07913925498723984, | |
| "rewards/rejected": -0.07037414610385895, | |
| "step": 38 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 26.369901657104492, | |
| "debug/policy_chosen_logps": -418.9932556152344, | |
| "debug/policy_rejected_logits": 24.119754791259766, | |
| "debug/policy_rejected_logps": -405.0431213378906, | |
| "debug/reference_chosen_logps": -421.83203125, | |
| "debug/reference_rejected_logps": -404.04913330078125, | |
| "epoch": 0.8297872340425532, | |
| "grad_norm": 5.464322991896784, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 26.369901657104492, | |
| "logits/rejected": 24.119754791259766, | |
| "logps/chosen": -418.9932556152344, | |
| "logps/rejected": -405.0431213378906, | |
| "loss": 0.4666, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.028387565165758133, | |
| "rewards/margins": 0.03832760080695152, | |
| "rewards/rejected": -0.009940031915903091, | |
| "step": 39 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 31.222116470336914, | |
| "debug/policy_chosen_logps": -408.92315673828125, | |
| "debug/policy_rejected_logits": 30.565526962280273, | |
| "debug/policy_rejected_logps": -438.00384521484375, | |
| "debug/reference_chosen_logps": -410.3876037597656, | |
| "debug/reference_rejected_logps": -428.6034851074219, | |
| "epoch": 0.851063829787234, | |
| "grad_norm": 5.087286245449022, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 31.222116470336914, | |
| "logits/rejected": 30.565526962280273, | |
| "logps/chosen": -408.92315673828125, | |
| "logps/rejected": -438.00384521484375, | |
| "loss": 0.4691, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.014644507318735123, | |
| "rewards/margins": 0.10864795744419098, | |
| "rewards/rejected": -0.09400344640016556, | |
| "step": 40 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.859107971191406, | |
| "debug/policy_chosen_logps": -403.02581787109375, | |
| "debug/policy_rejected_logits": 23.891035079956055, | |
| "debug/policy_rejected_logps": -395.781005859375, | |
| "debug/reference_chosen_logps": -409.97967529296875, | |
| "debug/reference_rejected_logps": -398.636474609375, | |
| "epoch": 0.8723404255319149, | |
| "grad_norm": 4.953135237737809, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.859107971191406, | |
| "logits/rejected": 23.891035079956055, | |
| "logps/chosen": -403.02581787109375, | |
| "logps/rejected": -395.781005859375, | |
| "loss": 0.458, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.06953833997249603, | |
| "rewards/margins": 0.04098331928253174, | |
| "rewards/rejected": 0.02855503186583519, | |
| "step": 41 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 31.02197265625, | |
| "debug/policy_chosen_logps": -424.50299072265625, | |
| "debug/policy_rejected_logits": 29.632427215576172, | |
| "debug/policy_rejected_logps": -422.94561767578125, | |
| "debug/reference_chosen_logps": -429.08648681640625, | |
| "debug/reference_rejected_logps": -424.3019104003906, | |
| "epoch": 0.8936170212765957, | |
| "grad_norm": 5.514556663745466, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 31.02197265625, | |
| "logits/rejected": 29.632427215576172, | |
| "logps/chosen": -424.50299072265625, | |
| "logps/rejected": -422.94561767578125, | |
| "loss": 0.4735, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.04583461582660675, | |
| "rewards/margins": 0.032271310687065125, | |
| "rewards/rejected": 0.01356330793350935, | |
| "step": 42 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 32.29981231689453, | |
| "debug/policy_chosen_logps": -437.9972229003906, | |
| "debug/policy_rejected_logits": 30.15468978881836, | |
| "debug/policy_rejected_logps": -440.30535888671875, | |
| "debug/reference_chosen_logps": -436.08892822265625, | |
| "debug/reference_rejected_logps": -434.7149353027344, | |
| "epoch": 0.9148936170212766, | |
| "grad_norm": 4.808288177536615, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 32.29981231689453, | |
| "logits/rejected": 30.15468978881836, | |
| "logps/chosen": -437.9972229003906, | |
| "logps/rejected": -440.30535888671875, | |
| "loss": 0.4598, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.01908310130238533, | |
| "rewards/margins": 0.03682101517915726, | |
| "rewards/rejected": -0.055904120206832886, | |
| "step": 43 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 26.440486907958984, | |
| "debug/policy_chosen_logps": -391.61572265625, | |
| "debug/policy_rejected_logits": 29.678592681884766, | |
| "debug/policy_rejected_logps": -430.9735412597656, | |
| "debug/reference_chosen_logps": -396.64862060546875, | |
| "debug/reference_rejected_logps": -434.947998046875, | |
| "epoch": 0.9361702127659575, | |
| "grad_norm": 5.150941817603919, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 26.440486907958984, | |
| "logits/rejected": 29.678592681884766, | |
| "logps/chosen": -391.61572265625, | |
| "logps/rejected": -430.9735412597656, | |
| "loss": 0.4568, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.050329361110925674, | |
| "rewards/margins": 0.010584792122244835, | |
| "rewards/rejected": 0.03974456712603569, | |
| "step": 44 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 29.451526641845703, | |
| "debug/policy_chosen_logps": -425.8896484375, | |
| "debug/policy_rejected_logits": 32.46401596069336, | |
| "debug/policy_rejected_logps": -428.1052551269531, | |
| "debug/reference_chosen_logps": -424.65936279296875, | |
| "debug/reference_rejected_logps": -427.16961669921875, | |
| "epoch": 0.9574468085106383, | |
| "grad_norm": 5.012447008649623, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 29.451526641845703, | |
| "logits/rejected": 32.46401596069336, | |
| "logps/chosen": -425.8896484375, | |
| "logps/rejected": -428.1052551269531, | |
| "loss": 0.4777, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.012303046882152557, | |
| "rewards/margins": -0.0029468159191310406, | |
| "rewards/rejected": -0.009356231428682804, | |
| "step": 45 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 31.168346405029297, | |
| "debug/policy_chosen_logps": -426.1267395019531, | |
| "debug/policy_rejected_logits": 29.51166534423828, | |
| "debug/policy_rejected_logps": -444.91766357421875, | |
| "debug/reference_chosen_logps": -429.6617736816406, | |
| "debug/reference_rejected_logps": -439.4256591796875, | |
| "epoch": 0.9787234042553191, | |
| "grad_norm": 5.030625016447312, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 31.168346405029297, | |
| "logits/rejected": 29.51166534423828, | |
| "logps/chosen": -426.1267395019531, | |
| "logps/rejected": -444.91766357421875, | |
| "loss": 0.4647, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.03535018861293793, | |
| "rewards/margins": 0.09027023613452911, | |
| "rewards/rejected": -0.05492004379630089, | |
| "step": 46 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 28.43193244934082, | |
| "debug/policy_chosen_logps": -419.4750671386719, | |
| "debug/policy_rejected_logits": 27.273754119873047, | |
| "debug/policy_rejected_logps": -438.4751892089844, | |
| "debug/reference_chosen_logps": -421.2059326171875, | |
| "debug/reference_rejected_logps": -437.2878723144531, | |
| "epoch": 1.0, | |
| "grad_norm": 5.486914446149956, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 28.43193244934082, | |
| "logits/rejected": 27.273754119873047, | |
| "logps/chosen": -419.4750671386719, | |
| "logps/rejected": -438.4751892089844, | |
| "loss": 0.4596, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.01730876788496971, | |
| "rewards/margins": 0.02918224036693573, | |
| "rewards/rejected": -0.011873474344611168, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 47, | |
| "total_flos": 0.0, | |
| "train_loss": 0.48215872493196044, | |
| "train_runtime": 474.4995, | |
| "train_samples_per_second": 6.327, | |
| "train_steps_per_second": 0.099 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 47, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |