diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,4835 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 73.65079365079364,
+  "eval_steps": 40,
+  "global_step": 1160,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.25,
+      "grad_norm": 7.928347587585449,
+      "learning_rate": 7.5e-08,
+      "logits/chosen": -2.7277705669403076,
+      "logits/rejected": -2.7679762840270996,
+      "logps/chosen": -128.4798583984375,
+      "logps/rejected": -98.36178588867188,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.109375,
+      "rewards/chosen": -7.17043731128797e-05,
+      "rewards/margins": 5.325676465872675e-05,
+      "rewards/rejected": -0.0001249611668754369,
+      "step": 4
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 9.242063522338867,
+      "learning_rate": 1.75e-07,
+      "logits/chosen": -2.6805009841918945,
+      "logits/rejected": -2.712625026702881,
+      "logps/chosen": -138.11959838867188,
+      "logps/rejected": -96.05882263183594,
+      "loss": 0.693,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": 0.0008271098486147821,
+      "rewards/margins": 0.00034989125560969114,
+      "rewards/rejected": 0.00047721865121275187,
+      "step": 8
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 8.156970977783203,
+      "learning_rate": 2.75e-07,
+      "logits/chosen": -2.7695400714874268,
+      "logits/rejected": -2.7420883178710938,
+      "logps/chosen": -133.04307556152344,
+      "logps/rejected": -104.19400024414062,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": 0.0034509659744799137,
+      "rewards/margins": 0.0025641382671892643,
+      "rewards/rejected": 0.0008868275908753276,
+      "step": 12
+    },
+    {
+      "epoch": 1.02,
+      "grad_norm": 8.723258018493652,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": -2.6869003772735596,
+      "logits/rejected": -2.7081785202026367,
+      "logps/chosen": -126.2979736328125,
+      "logps/rejected": -98.4257583618164,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.011365305632352829,
+      "rewards/margins": 0.008443659171462059,
+      "rewards/rejected": 0.0029216469265520573,
+      "step": 16
+    },
+    {
+      "epoch": 1.27,
+      "grad_norm": 10.34566593170166,
+      "learning_rate": 4.7499999999999995e-07,
+      "logits/chosen": -2.704402208328247,
+      "logits/rejected": -2.764737367630005,
+      "logps/chosen": -125.91941833496094,
+      "logps/rejected": -99.02344512939453,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.02058127522468567,
+      "rewards/margins": 0.020110297948122025,
+      "rewards/rejected": 0.00047097797505557537,
+      "step": 20
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 9.291991233825684,
+      "learning_rate": 4.999985633126757e-07,
+      "logits/chosen": -2.716165781021118,
+      "logits/rejected": -2.6845083236694336,
+      "logps/chosen": -137.2718505859375,
+      "logps/rejected": -97.50873565673828,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.03298673778772354,
+      "rewards/margins": 0.029706846922636032,
+      "rewards/rejected": 0.0032798887696117163,
+      "step": 24
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 9.161444664001465,
+      "learning_rate": 4.999921780689761e-07,
+      "logits/chosen": -2.735781192779541,
+      "logits/rejected": -2.7485365867614746,
+      "logps/chosen": -128.5098876953125,
+      "logps/rejected": -90.1419906616211,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.04758748784661293,
+      "rewards/margins": 0.04847026616334915,
+      "rewards/rejected": -0.0008827749406918883,
+      "step": 28
+    },
+    {
+      "epoch": 2.03,
+      "grad_norm": 8.495955467224121,
+      "learning_rate": 4.99980684767309e-07,
+      "logits/chosen": -2.7206833362579346,
+      "logits/rejected": -2.750042676925659,
+      "logps/chosen": -133.54827880859375,
+      "logps/rejected": -108.23342895507812,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.0631466805934906,
+      "rewards/margins": 0.05378420650959015,
+      "rewards/rejected": 0.009362474083900452,
+      "step": 32
+    },
+    {
+      "epoch": 2.29,
+      "grad_norm": 8.951277732849121,
+      "learning_rate": 4.999640836425158e-07,
+      "logits/chosen": -2.6706807613372803,
+      "logits/rejected": -2.6151680946350098,
+      "logps/chosen": -136.55226135253906,
+      "logps/rejected": -97.76315307617188,
+      "loss": 0.6578,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.08149851858615875,
+      "rewards/margins": 0.07302143424749374,
+      "rewards/rejected": 0.008477086201310158,
+      "step": 36
+    },
+    {
+      "epoch": 2.54,
+      "grad_norm": 8.608317375183105,
+      "learning_rate": 4.99942375033805e-07,
+      "logits/chosen": -2.7676639556884766,
+      "logits/rejected": -2.833590030670166,
+      "logps/chosen": -133.7703857421875,
+      "logps/rejected": -106.12245178222656,
+      "loss": 0.6488,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10222765803337097,
+      "rewards/margins": 0.09170522540807724,
+      "rewards/rejected": 0.010522443801164627,
+      "step": 40
+    },
+    {
+      "epoch": 2.54,
+      "eval_logits/chosen": -2.806352138519287,
+      "eval_logits/rejected": -2.8898861408233643,
+      "eval_logps/chosen": -133.14093017578125,
+      "eval_logps/rejected": -97.8843002319336,
+      "eval_loss": 0.6592782139778137,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 0.09283973276615143,
+      "eval_rewards/margins": 0.07038124650716782,
+      "eval_rewards/rejected": 0.022458484396338463,
+      "eval_runtime": 19.3683,
+      "eval_samples_per_second": 1.446,
+      "eval_steps_per_second": 1.446,
+      "step": 40
+    },
+    {
+      "epoch": 2.79,
+      "grad_norm": 9.09485912322998,
+      "learning_rate": 4.999155593847457e-07,
+      "logits/chosen": -2.7024385929107666,
+      "logits/rejected": -2.7564737796783447,
+      "logps/chosen": -123.412353515625,
+      "logps/rejected": -91.91966247558594,
+      "loss": 0.6347,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.114817313849926,
+      "rewards/margins": 0.12200291454792023,
+      "rewards/rejected": -0.007185595575720072,
+      "step": 44
+    },
+    {
+      "epoch": 3.05,
+      "grad_norm": 9.453149795532227,
+      "learning_rate": 4.998836372432589e-07,
+      "logits/chosen": -2.736665725708008,
+      "logits/rejected": -2.757841110229492,
+      "logps/chosen": -127.0240478515625,
+      "logps/rejected": -103.66780853271484,
+      "loss": 0.636,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.13144327700138092,
+      "rewards/margins": 0.11972987651824951,
+      "rewards/rejected": 0.011713397689163685,
+      "step": 48
+    },
+    {
+      "epoch": 3.3,
+      "grad_norm": 8.674492835998535,
+      "learning_rate": 4.998466092616052e-07,
+      "logits/chosen": -2.710110664367676,
+      "logits/rejected": -2.7205581665039062,
+      "logps/chosen": -131.0464324951172,
+      "logps/rejected": -99.71279907226562,
+      "loss": 0.6166,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.15897569060325623,
+      "rewards/margins": 0.16229701042175293,
+      "rewards/rejected": -0.003321333322674036,
+      "step": 52
+    },
+    {
+      "epoch": 3.56,
+      "grad_norm": 9.434328079223633,
+      "learning_rate": 4.99804476196373e-07,
+      "logits/chosen": -2.685814380645752,
+      "logits/rejected": -2.736269474029541,
+      "logps/chosen": -125.750732421875,
+      "logps/rejected": -93.69428253173828,
+      "loss": 0.6131,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.16781488060951233,
+      "rewards/margins": 0.16998659074306488,
+      "rewards/rejected": -0.002171688713133335,
+      "step": 56
+    },
+    {
+      "epoch": 3.81,
+      "grad_norm": 8.643455505371094,
+      "learning_rate": 4.997572389084615e-07,
+      "logits/chosen": -2.7429587841033936,
+      "logits/rejected": -2.75490140914917,
+      "logps/chosen": -131.4405517578125,
+      "logps/rejected": -99.93754577636719,
+      "loss": 0.614,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.18873561918735504,
+      "rewards/margins": 0.17132429778575897,
+      "rewards/rejected": 0.01741132140159607,
+      "step": 60
+    },
+    {
+      "epoch": 4.06,
+      "grad_norm": 8.412778854370117,
+      "learning_rate": 4.997048983630643e-07,
+      "logits/chosen": -2.6886143684387207,
+      "logits/rejected": -2.710732936859131,
+      "logps/chosen": -123.77167510986328,
+      "logps/rejected": -97.30846405029297,
+      "loss": 0.6099,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19313779473304749,
+      "rewards/margins": 0.1793561577796936,
+      "rewards/rejected": 0.013781649991869926,
+      "step": 64
+    },
+    {
+      "epoch": 4.32,
+      "grad_norm": 7.851288795471191,
+      "learning_rate": 4.99647455629649e-07,
+      "logits/chosen": -2.744528293609619,
+      "logits/rejected": -2.7117881774902344,
+      "logps/chosen": -136.25863647460938,
+      "logps/rejected": -95.11305236816406,
+      "loss": 0.5968,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.21463464200496674,
+      "rewards/margins": 0.20793242752552032,
+      "rewards/rejected": 0.0067022331058979034,
+      "step": 68
+    },
+    {
+      "epoch": 4.57,
+      "grad_norm": 9.278959274291992,
+      "learning_rate": 4.995849118819353e-07,
+      "logits/chosen": -2.7611894607543945,
+      "logits/rejected": -2.7447633743286133,
+      "logps/chosen": -134.3462677001953,
+      "logps/rejected": -103.75216674804688,
+      "loss": 0.5828,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.25433310866355896,
+      "rewards/margins": 0.24298511445522308,
+      "rewards/rejected": 0.011347985826432705,
+      "step": 72
+    },
+    {
+      "epoch": 4.83,
+      "grad_norm": 7.795536994934082,
+      "learning_rate": 4.995172683978719e-07,
+      "logits/chosen": -2.698429584503174,
+      "logits/rejected": -2.7458348274230957,
+      "logps/chosen": -127.65770721435547,
+      "logps/rejected": -104.23495483398438,
+      "loss": 0.5796,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.25434932112693787,
+      "rewards/margins": 0.25114190578460693,
+      "rewards/rejected": 0.0032073920592665672,
+      "step": 76
+    },
+    {
+      "epoch": 5.08,
+      "grad_norm": 7.788575649261475,
+      "learning_rate": 4.994445265596091e-07,
+      "logits/chosen": -2.6734955310821533,
+      "logits/rejected": -2.7292165756225586,
+      "logps/chosen": -123.53501892089844,
+      "logps/rejected": -95.23420715332031,
+      "loss": 0.5676,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2950035631656647,
+      "rewards/margins": 0.2805778682231903,
+      "rewards/rejected": 0.014425676316022873,
+      "step": 80
+    },
+    {
+      "epoch": 5.08,
+      "eval_logits/chosen": -2.8074097633361816,
+      "eval_logits/rejected": -2.890533685684204,
+      "eval_logps/chosen": -131.55104064941406,
+      "eval_logps/rejected": -97.56663513183594,
+      "eval_loss": 0.6045485734939575,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 0.2518289387226105,
+      "eval_rewards/margins": 0.1976027488708496,
+      "eval_rewards/rejected": 0.05422618240118027,
+      "eval_runtime": 19.3586,
+      "eval_samples_per_second": 1.446,
+      "eval_steps_per_second": 1.446,
+      "step": 80
+    },
+    {
+      "epoch": 5.33,
+      "grad_norm": 6.709670543670654,
+      "learning_rate": 4.993666878534718e-07,
+      "logits/chosen": -2.736870527267456,
+      "logits/rejected": -2.778140068054199,
+      "logps/chosen": -128.86972045898438,
+      "logps/rejected": -104.70977020263672,
+      "loss": 0.5637,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.31255263090133667,
+      "rewards/margins": 0.2908375561237335,
+      "rewards/rejected": 0.021715058013796806,
+      "step": 84
+    },
+    {
+      "epoch": 5.59,
+      "grad_norm": 8.178019523620605,
+      "learning_rate": 4.99283753869928e-07,
+      "logits/chosen": -2.686039447784424,
+      "logits/rejected": -2.681716203689575,
+      "logps/chosen": -129.52798461914062,
+      "logps/rejected": -97.80433654785156,
+      "loss": 0.5475,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32084640860557556,
+      "rewards/margins": 0.33009591698646545,
+      "rewards/rejected": -0.00924946740269661,
+      "step": 88
+    },
+    {
+      "epoch": 5.84,
+      "grad_norm": 7.515178680419922,
+      "learning_rate": 4.991957263035573e-07,
+      "logits/chosen": -2.763765811920166,
+      "logits/rejected": -2.7660324573516846,
+      "logps/chosen": -128.16851806640625,
+      "logps/rejected": -95.72579956054688,
+      "loss": 0.5493,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3080848455429077,
+      "rewards/margins": 0.32350555062294006,
+      "rewards/rejected": -0.015420722775161266,
+      "step": 92
+    },
+    {
+      "epoch": 6.1,
+      "grad_norm": 8.599517822265625,
+      "learning_rate": 4.991026069530156e-07,
+      "logits/chosen": -2.7107338905334473,
+      "logits/rejected": -2.745981216430664,
+      "logps/chosen": -131.5907745361328,
+      "logps/rejected": -102.65729522705078,
+      "loss": 0.5568,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.36698901653289795,
+      "rewards/margins": 0.3141450881958008,
+      "rewards/rejected": 0.05284389108419418,
+      "step": 96
+    },
+    {
+      "epoch": 6.35,
+      "grad_norm": 7.087887287139893,
+      "learning_rate": 4.990043977209984e-07,
+      "logits/chosen": -2.75876522064209,
+      "logits/rejected": -2.800281047821045,
+      "logps/chosen": -124.87574005126953,
+      "logps/rejected": -99.60267639160156,
+      "loss": 0.5332,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3760470151901245,
+      "rewards/margins": 0.37043848633766174,
+      "rewards/rejected": 0.00560858054086566,
+      "step": 100
+    },
+    {
+      "epoch": 6.6,
+      "grad_norm": 6.458583831787109,
+      "learning_rate": 4.989011006142024e-07,
+      "logits/chosen": -2.70744252204895,
+      "logits/rejected": -2.7285239696502686,
+      "logps/chosen": -129.41262817382812,
+      "logps/rejected": -98.60986328125,
+      "loss": 0.5286,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3909401297569275,
+      "rewards/margins": 0.3845981955528259,
+      "rewards/rejected": 0.006341893225908279,
+      "step": 104
+    },
+    {
+      "epoch": 6.86,
+      "grad_norm": 8.084393501281738,
+      "learning_rate": 4.987927177432842e-07,
+      "logits/chosen": -2.731127977371216,
+      "logits/rejected": -2.7352988719940186,
+      "logps/chosen": -123.85149383544922,
+      "logps/rejected": -97.99423217773438,
+      "loss": 0.544,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.38825392723083496,
+      "rewards/margins": 0.34452566504478455,
+      "rewards/rejected": 0.043728262186050415,
+      "step": 108
+    },
+    {
+      "epoch": 7.11,
+      "grad_norm": 7.936505317687988,
+      "learning_rate": 4.987080943856886e-07,
+      "logits/chosen": -2.6931982040405273,
+      "logits/rejected": -2.683288812637329,
+      "logps/chosen": -130.97227478027344,
+      "logps/rejected": -93.71176147460938,
+      "loss": 0.4903,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.45238468050956726,
+      "rewards/margins": 0.48204928636550903,
+      "rewards/rejected": -0.029664600268006325,
+      "step": 112
+    },
+    {
+      "epoch": 7.37,
+      "grad_norm": 6.827272415161133,
+      "learning_rate": 4.985908168188602e-07,
+      "logits/chosen": -2.697653293609619,
+      "logits/rejected": -2.7467117309570312,
+      "logps/chosen": -119.16637420654297,
+      "logps/rejected": -95.0647201538086,
+      "loss": 0.5088,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.4183903634548187,
+      "rewards/margins": 0.43436184525489807,
+      "rewards/rejected": -0.015971507877111435,
+      "step": 116
+    },
+    {
+      "epoch": 7.62,
+      "grad_norm": 6.599144458770752,
+      "learning_rate": 4.984684598278982e-07,
+      "logits/chosen": -2.7136449813842773,
+      "logits/rejected": -2.7430026531219482,
+      "logps/chosen": -126.032958984375,
+      "logps/rejected": -102.69947814941406,
+      "loss": 0.5231,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4575308859348297,
+      "rewards/margins": 0.40544256567955017,
+      "rewards/rejected": 0.052088312804698944,
+      "step": 120
+    },
+    {
+      "epoch": 7.62,
+      "eval_logits/chosen": -2.807668447494507,
+      "eval_logits/rejected": -2.8901402950286865,
+      "eval_logps/chosen": -129.97122192382812,
+      "eval_logps/rejected": -97.28166961669922,
+      "eval_loss": 0.5570675134658813,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 0.4098115563392639,
+      "eval_rewards/margins": 0.3270883560180664,
+      "eval_rewards/rejected": 0.0827232226729393,
+      "eval_runtime": 19.4265,
+      "eval_samples_per_second": 1.441,
+      "eval_steps_per_second": 1.441,
+      "step": 120
+    },
+    {
+      "epoch": 7.87,
+      "grad_norm": 7.769857883453369,
+      "learning_rate": 4.983410259129073e-07,
+      "logits/chosen": -2.7467093467712402,
+      "logits/rejected": -2.7454986572265625,
+      "logps/chosen": -134.60739135742188,
+      "logps/rejected": -103.20536804199219,
+      "loss": 0.4995,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.4985056221485138,
+      "rewards/margins": 0.4699307978153229,
+      "rewards/rejected": 0.028574790805578232,
+      "step": 124
+    },
+    {
+      "epoch": 8.13,
+      "grad_norm": 7.224113941192627,
+      "learning_rate": 4.982085176777285e-07,
+      "logits/chosen": -2.725543975830078,
+      "logits/rejected": -2.7536404132843018,
+      "logps/chosen": -121.35929870605469,
+      "logps/rejected": -97.49042510986328,
+      "loss": 0.4988,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.4724448025226593,
+      "rewards/margins": 0.4719679057598114,
+      "rewards/rejected": 0.0004768962971866131,
+      "step": 128
+    },
+    {
+      "epoch": 8.38,
+      "grad_norm": 5.916596412658691,
+      "learning_rate": 4.980709378298851e-07,
+      "logits/chosen": -2.7414817810058594,
+      "logits/rejected": -2.7378463745117188,
+      "logps/chosen": -119.76469421386719,
+      "logps/rejected": -92.40216827392578,
+      "loss": 0.4946,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.48038381338119507,
+      "rewards/margins": 0.4809207320213318,
+      "rewards/rejected": -0.0005369335412979126,
+      "step": 132
+    },
+    {
+      "epoch": 8.63,
+      "grad_norm": 6.414724826812744,
+      "learning_rate": 4.979282891805286e-07,
+      "logits/chosen": -2.7029452323913574,
+      "logits/rejected": -2.737753391265869,
+      "logps/chosen": -130.59829711914062,
+      "logps/rejected": -99.5873794555664,
+      "loss": 0.4564,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5609064698219299,
+      "rewards/margins": 0.5927521586418152,
+      "rewards/rejected": -0.03184571862220764,
+      "step": 136
+    },
+    {
+      "epoch": 8.89,
+      "grad_norm": 7.006284236907959,
+      "learning_rate": 4.977805746443806e-07,
+      "logits/chosen": -2.719618320465088,
+      "logits/rejected": -2.7250468730926514,
+      "logps/chosen": -128.04678344726562,
+      "logps/rejected": -101.81006622314453,
+      "loss": 0.4761,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5477940440177917,
+      "rewards/margins": 0.5303957462310791,
+      "rewards/rejected": 0.017398254945874214,
+      "step": 140
+    },
+    {
+      "epoch": 9.14,
+      "grad_norm": 7.040762901306152,
+      "learning_rate": 4.97627797239673e-07,
+      "logits/chosen": -2.6895217895507812,
+      "logits/rejected": -2.7162435054779053,
+      "logps/chosen": -133.3381805419922,
+      "logps/rejected": -102.25975799560547,
+      "loss": 0.4403,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.6185128688812256,
+      "rewards/margins": 0.6498669981956482,
+      "rewards/rejected": -0.03135409206151962,
+      "step": 144
+    },
+    {
+      "epoch": 9.4,
+      "grad_norm": 6.93245267868042,
+      "learning_rate": 4.974699600880869e-07,
+      "logits/chosen": -2.6911263465881348,
+      "logits/rejected": -2.7075552940368652,
+      "logps/chosen": -126.95785522460938,
+      "logps/rejected": -102.03314971923828,
+      "loss": 0.4587,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.573100745677948,
+      "rewards/margins": 0.5804403424263,
+      "rewards/rejected": -0.0073395660147070885,
+      "step": 148
+    },
+    {
+      "epoch": 9.65,
+      "grad_norm": 7.084275245666504,
+      "learning_rate": 4.973070664146885e-07,
+      "logits/chosen": -2.7053210735321045,
+      "logits/rejected": -2.7612156867980957,
+      "logps/chosen": -119.92796325683594,
+      "logps/rejected": -97.49836730957031,
+      "loss": 0.483,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5761281251907349,
+      "rewards/margins": 0.5190825462341309,
+      "rewards/rejected": 0.057045597583055496,
+      "step": 152
+    },
+    {
+      "epoch": 9.9,
+      "grad_norm": 5.3888258934021,
+      "learning_rate": 4.971391195478631e-07,
+      "logits/chosen": -2.750107765197754,
+      "logits/rejected": -2.7442452907562256,
+      "logps/chosen": -128.85630798339844,
+      "logps/rejected": -96.89845275878906,
+      "loss": 0.4517,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.6420415043830872,
+      "rewards/margins": 0.6197397708892822,
+      "rewards/rejected": 0.022301698103547096,
+      "step": 156
+    },
+    {
+      "epoch": 10.16,
+      "grad_norm": 6.74313497543335,
+      "learning_rate": 4.969661229192477e-07,
+      "logits/chosen": -2.7252540588378906,
+      "logits/rejected": -2.709249258041382,
+      "logps/chosen": -126.21170043945312,
+      "logps/rejected": -105.25879669189453,
+      "loss": 0.4607,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.6133719682693481,
+      "rewards/margins": 0.6076129674911499,
+      "rewards/rejected": 0.005758981220424175,
+      "step": 160
+    },
+    {
+      "epoch": 10.16,
+      "eval_logits/chosen": -2.807128429412842,
+      "eval_logits/rejected": -2.8889076709747314,
+      "eval_logps/chosen": -128.48606872558594,
+      "eval_logps/rejected": -97.06517028808594,
+      "eval_loss": 0.5175719857215881,
+      "eval_rewards/accuracies": 0.8214285969734192,
+      "eval_rewards/chosen": 0.5583271384239197,
+      "eval_rewards/margins": 0.4539553225040436,
+      "eval_rewards/rejected": 0.10437185317277908,
+      "eval_runtime": 19.3848,
+      "eval_samples_per_second": 1.444,
+      "eval_steps_per_second": 1.444,
+      "step": 160
+    },
+    {
+      "epoch": 10.41,
+      "grad_norm": 5.443469524383545,
+      "learning_rate": 4.967880800636598e-07,
+      "logits/chosen": -2.7496747970581055,
+      "logits/rejected": -2.768813133239746,
+      "logps/chosen": -122.11544799804688,
+      "logps/rejected": -94.54540252685547,
+      "loss": 0.436,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6616504788398743,
+      "rewards/margins": 0.67127525806427,
+      "rewards/rejected": -0.009624744765460491,
+      "step": 164
+    },
+    {
+      "epoch": 10.67,
+      "grad_norm": 5.551913738250732,
+      "learning_rate": 4.966049946190263e-07,
+      "logits/chosen": -2.7052814960479736,
+      "logits/rejected": -2.7216036319732666,
+      "logps/chosen": -131.8173065185547,
+      "logps/rejected": -102.79641723632812,
+      "loss": 0.4326,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.7013098001480103,
+      "rewards/margins": 0.6718720197677612,
+      "rewards/rejected": 0.02943783439695835,
+      "step": 168
+    },
+    {
+      "epoch": 10.92,
+      "grad_norm": 5.826549530029297,
+      "learning_rate": 4.964168703263086e-07,
+      "logits/chosen": -2.701007604598999,
+      "logits/rejected": -2.741464614868164,
+      "logps/chosen": -116.08711242675781,
+      "logps/rejected": -94.73246765136719,
+      "loss": 0.446,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.6390998363494873,
+      "rewards/margins": 0.6307864785194397,
+      "rewards/rejected": 0.008313396014273167,
+      "step": 172
+    },
+    {
+      "epoch": 11.17,
+      "grad_norm": 6.066051483154297,
+      "learning_rate": 4.962237110294259e-07,
+      "logits/chosen": -2.7041149139404297,
+      "logits/rejected": -2.731684684753418,
+      "logps/chosen": -124.84115600585938,
+      "logps/rejected": -103.8166732788086,
+      "loss": 0.4294,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.6835889220237732,
+      "rewards/margins": 0.6921644806861877,
+      "rewards/rejected": -0.00857558287680149,
+      "step": 176
+    },
+    {
+      "epoch": 11.43,
+      "grad_norm": 5.8145012855529785,
+      "learning_rate": 4.960255206751773e-07,
+      "logits/chosen": -2.7352046966552734,
+      "logits/rejected": -2.7856600284576416,
+      "logps/chosen": -123.47644805908203,
+      "logps/rejected": -93.96688079833984,
+      "loss": 0.4056,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.7594637274742126,
+      "rewards/margins": 0.7790957093238831,
+      "rewards/rejected": -0.019631966948509216,
+      "step": 180
+    },
+    {
+      "epoch": 11.68,
+      "grad_norm": 5.962335109710693,
+      "learning_rate": 4.958223033131609e-07,
+      "logits/chosen": -2.740914821624756,
+      "logits/rejected": -2.7585391998291016,
+      "logps/chosen": -124.35005950927734,
+      "logps/rejected": -105.05398559570312,
+      "loss": 0.4293,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.6812430620193481,
+      "rewards/margins": 0.6962765455245972,
+      "rewards/rejected": -0.015033497475087643,
+      "step": 184
+    },
+    {
+      "epoch": 11.94,
+      "grad_norm": 5.529379367828369,
+      "learning_rate": 4.956140630956907e-07,
+      "logits/chosen": -2.7024734020233154,
+      "logits/rejected": -2.6746203899383545,
+      "logps/chosen": -125.29593658447266,
+      "logps/rejected": -94.42231750488281,
+      "loss": 0.3963,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.7892162799835205,
+      "rewards/margins": 0.8050926327705383,
+      "rewards/rejected": -0.01587628945708275,
+      "step": 188
+    },
+    {
+      "epoch": 12.19,
+      "grad_norm": 5.774294376373291,
+      "learning_rate": 4.954008042777125e-07,
+      "logits/chosen": -2.7662813663482666,
+      "logits/rejected": -2.8167452812194824,
+      "logps/chosen": -124.10614776611328,
+      "logps/rejected": -94.9889907836914,
+      "loss": 0.3728,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.8182777166366577,
+      "rewards/margins": 0.8937400579452515,
+      "rewards/rejected": -0.07546230405569077,
+      "step": 192
+    },
+    {
+      "epoch": 12.44,
+      "grad_norm": 4.990906715393066,
+      "learning_rate": 4.951825312167159e-07,
+      "logits/chosen": -2.7352771759033203,
+      "logits/rejected": -2.7206149101257324,
+      "logps/chosen": -116.98539733886719,
+      "logps/rejected": -96.68843078613281,
+      "loss": 0.4435,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.7208267450332642,
+      "rewards/margins": 0.6662319302558899,
+      "rewards/rejected": 0.05459484085440636,
+      "step": 196
+    },
+    {
+      "epoch": 12.7,
+      "grad_norm": 7.139024257659912,
+      "learning_rate": 4.949592483726464e-07,
+      "logits/chosen": -2.728516101837158,
+      "logits/rejected": -2.732177257537842,
+      "logps/chosen": -125.51838684082031,
+      "logps/rejected": -103.7790298461914,
+      "loss": 0.391,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.7955451011657715,
+      "rewards/margins": 0.8273122310638428,
+      "rewards/rejected": -0.03176717832684517,
+      "step": 200
+    },
+    {
+      "epoch": 12.7,
+      "eval_logits/chosen": -2.8053383827209473,
+      "eval_logits/rejected": -2.8861937522888184,
+      "eval_logps/chosen": -127.08057403564453,
+      "eval_logps/rejected": -96.8802261352539,
+      "eval_loss": 0.4858916103839874,
+      "eval_rewards/accuracies": 0.8214285969734192,
+      "eval_rewards/chosen": 0.6988765597343445,
+      "eval_rewards/margins": 0.5760089755058289,
+      "eval_rewards/rejected": 0.12286762148141861,
+      "eval_runtime": 19.4173,
+      "eval_samples_per_second": 1.442,
+      "eval_steps_per_second": 1.442,
+      "step": 200
+    },
+    {
+      "epoch": 12.95,
+      "grad_norm": 5.450782299041748,
+      "learning_rate": 4.947309603078138e-07,
+      "logits/chosen": -2.6719353199005127,
+      "logits/rejected": -2.7134647369384766,
+      "logps/chosen": -124.22660827636719,
+      "logps/rejected": -99.73461151123047,
+      "loss": 0.4062,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.7935754060745239,
+      "rewards/margins": 0.783437192440033,
+      "rewards/rejected": 0.010138224810361862,
+      "step": 204
+    },
+    {
+      "epoch": 13.21,
+      "grad_norm": 5.752992153167725,
+      "learning_rate": 4.944976716867984e-07,
+      "logits/chosen": -2.7026755809783936,
+      "logits/rejected": -2.698920726776123,
+      "logps/chosen": -123.63475036621094,
+      "logps/rejected": -107.06332397460938,
+      "loss": 0.3952,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.8116875886917114,
+      "rewards/margins": 0.8335397839546204,
+      "rewards/rejected": -0.02185220457613468,
+      "step": 208
+    },
+    {
+      "epoch": 13.46,
+      "grad_norm": 5.456747531890869,
+      "learning_rate": 4.942593872763566e-07,
+      "logits/chosen": -2.701968193054199,
+      "logits/rejected": -2.786785840988159,
+      "logps/chosen": -124.91243743896484,
+      "logps/rejected": -101.95451354980469,
+      "loss": 0.3673,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.8831655979156494,
+      "rewards/margins": 0.9259040951728821,
+      "rewards/rejected": -0.04273851588368416,
+      "step": 212
+    },
+    {
+      "epoch": 13.71,
+      "grad_norm": 4.753533363342285,
+      "learning_rate": 4.940161119453231e-07,
+      "logits/chosen": -2.7109193801879883,
+      "logits/rejected": -2.6961448192596436,
+      "logps/chosen": -123.21333312988281,
+      "logps/rejected": -92.74653625488281,
+      "loss": 0.3832,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.8407853245735168,
+      "rewards/margins": 0.8698095083236694,
+      "rewards/rejected": -0.02902432158589363,
+      "step": 216
+    },
+    {
+      "epoch": 13.97,
+      "grad_norm": 5.374657154083252,
+      "learning_rate": 4.937678506645115e-07,
+      "logits/chosen": -2.7288832664489746,
+      "logits/rejected": -2.7349352836608887,
+      "logps/chosen": -123.1498031616211,
+      "logps/rejected": -95.77909851074219,
+      "loss": 0.3644,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.8925550580024719,
+      "rewards/margins": 0.9183064699172974,
+      "rewards/rejected": -0.025751333683729172,
+      "step": 220
+    },
+    {
+      "epoch": 14.22,
+      "grad_norm": 5.842701435089111,
+      "learning_rate": 4.935146085066125e-07,
+      "logits/chosen": -2.7030577659606934,
+      "logits/rejected": -2.7200875282287598,
+      "logps/chosen": -120.81951904296875,
+      "logps/rejected": -100.53494262695312,
+      "loss": 0.3581,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.9156714677810669,
+      "rewards/margins": 0.9704564809799194,
+      "rewards/rejected": -0.05478499457240105,
+      "step": 224
+    },
+    {
+      "epoch": 14.48,
+      "grad_norm": 5.020945072174072,
+      "learning_rate": 4.932563906460904e-07,
+      "logits/chosen": -2.722069501876831,
+      "logits/rejected": -2.731358051300049,
+      "logps/chosen": -123.65387725830078,
+      "logps/rejected": -103.83390808105469,
+      "loss": 0.391,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.8787897229194641,
+      "rewards/margins": 0.8743190765380859,
+      "rewards/rejected": 0.004470685496926308,
+      "step": 228
+    },
+    {
+      "epoch": 14.73,
+      "grad_norm": 5.041106700897217,
+      "learning_rate": 4.929932023590776e-07,
+      "logits/chosen": -2.735006093978882,
+      "logits/rejected": -2.7546191215515137,
+      "logps/chosen": -124.65998840332031,
+      "logps/rejected": -101.49640655517578,
+      "loss": 0.3586,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.9206160306930542,
+      "rewards/margins": 0.9452594518661499,
+      "rewards/rejected": -0.024643439799547195,
+      "step": 232
+    },
+    {
+      "epoch": 14.98,
+      "grad_norm": 5.318466663360596,
+      "learning_rate": 4.927250490232664e-07,
+      "logits/chosen": -2.6883015632629395,
+      "logits/rejected": -2.718212842941284,
+      "logps/chosen": -120.88870239257812,
+      "logps/rejected": -94.79769134521484,
+      "loss": 0.3564,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.9339860677719116,
+      "rewards/margins": 0.9901734590530396,
+      "rewards/rejected": -0.05618742108345032,
+      "step": 236
+    },
+    {
+      "epoch": 15.24,
+      "grad_norm": 5.438621997833252,
+      "learning_rate": 4.924519361177992e-07,
+      "logits/chosen": -2.7316794395446777,
+      "logits/rejected": -2.812800407409668,
+      "logps/chosen": -120.06542205810547,
+      "logps/rejected": -97.61270141601562,
+      "loss": 0.3471,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.9851706624031067,
+      "rewards/margins": 1.039989948272705,
+      "rewards/rejected": -0.05481935292482376,
+      "step": 240
+    },
+    {
+      "epoch": 15.24,
+      "eval_logits/chosen": -2.8022568225860596,
+      "eval_logits/rejected": -2.882326602935791,
+      "eval_logps/chosen": -125.87542724609375,
+      "eval_logps/rejected": -96.83832550048828,
+      "eval_loss": 0.46029001474380493,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 0.8193913102149963,
+      "eval_rewards/margins": 0.6923348307609558,
+      "eval_rewards/rejected": 0.12705650925636292,
+      "eval_runtime": 19.4065,
+      "eval_samples_per_second": 1.443,
+      "eval_steps_per_second": 1.443,
+      "step": 240
+    },
+    {
+      "epoch": 15.49,
+      "grad_norm": 5.301855087280273,
+      "learning_rate": 4.921738692231572e-07,
+      "logits/chosen": -2.661013126373291,
+      "logits/rejected": -2.666978359222412,
+      "logps/chosen": -121.69336700439453,
+      "logps/rejected": -101.86541748046875,
+      "loss": 0.3574,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.9682250022888184,
+      "rewards/margins": 0.9734476208686829,
+      "rewards/rejected": -0.0052225226536393166,
+      "step": 244
+    },
+    {
+      "epoch": 15.75,
+      "grad_norm": 4.478713512420654,
+      "learning_rate": 4.918908540210451e-07,
+      "logits/chosen": -2.702775239944458,
+      "logits/rejected": -2.7098357677459717,
+      "logps/chosen": -119.93941497802734,
+      "logps/rejected": -94.76625061035156,
+      "loss": 0.3517,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.8942417502403259,
+      "rewards/margins": 0.9958410263061523,
+      "rewards/rejected": -0.10159924626350403,
+      "step": 248
+    },
+    {
+      "epoch": 16.0,
+      "grad_norm": 4.398434162139893,
+      "learning_rate": 4.916028962942762e-07,
+      "logits/chosen": -2.7399990558624268,
+      "logits/rejected": -2.7476425170898438,
+      "logps/chosen": -124.2032470703125,
+      "logps/rejected": -101.86943054199219,
+      "loss": 0.3372,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.0369476079940796,
+      "rewards/margins": 1.0664609670639038,
+      "rewards/rejected": -0.029513416811823845,
+      "step": 252
+    },
+    {
+      "epoch": 16.25,
+      "grad_norm": 4.430797576904297,
+      "learning_rate": 4.913100019266536e-07,
+      "logits/chosen": -2.6950950622558594,
+      "logits/rejected": -2.7181689739227295,
+      "logps/chosen": -123.97085571289062,
+      "logps/rejected": -97.85747528076172,
+      "loss": 0.3226,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.0282835960388184,
+      "rewards/margins": 1.1182292699813843,
+      "rewards/rejected": -0.08994561433792114,
+      "step": 256
+    },
+    {
+      "epoch": 16.51,
+      "grad_norm": 6.383476734161377,
+      "learning_rate": 4.910121769028503e-07,
+      "logits/chosen": -2.7224864959716797,
+      "logits/rejected": -2.727613925933838,
+      "logps/chosen": -117.91517639160156,
+      "logps/rejected": -99.52539825439453,
+      "loss": 0.3658,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.9342272877693176,
+      "rewards/margins": 0.9777376651763916,
+      "rewards/rejected": -0.04351034015417099,
+      "step": 260
+    },
+    {
+      "epoch": 16.76,
+      "grad_norm": 4.453210353851318,
+      "learning_rate": 4.907094273082864e-07,
+      "logits/chosen": -2.7149100303649902,
+      "logits/rejected": -2.710146188735962,
+      "logps/chosen": -126.07495880126953,
+      "logps/rejected": -100.1856689453125,
+      "loss": 0.3162,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 1.1231064796447754,
+      "rewards/margins": 1.175417423248291,
+      "rewards/rejected": -0.05231098458170891,
+      "step": 264
+    },
+    {
+      "epoch": 17.02,
+      "grad_norm": 4.718712329864502,
+      "learning_rate": 4.904017593290055e-07,
+      "logits/chosen": -2.710063934326172,
+      "logits/rejected": -2.7535319328308105,
+      "logps/chosen": -117.01451873779297,
+      "logps/rejected": -99.95758819580078,
+      "loss": 0.322,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.033459186553955,
+      "rewards/margins": 1.1137871742248535,
+      "rewards/rejected": -0.08032786101102829,
+      "step": 268
+    },
+    {
+      "epoch": 17.27,
+      "grad_norm": 5.164140701293945,
+      "learning_rate": 4.900891792515478e-07,
+      "logits/chosen": -2.7342586517333984,
+      "logits/rejected": -2.7561776638031006,
+      "logps/chosen": -118.80335998535156,
+      "logps/rejected": -104.33666229248047,
+      "loss": 0.3314,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.078546404838562,
+      "rewards/margins": 1.0902469158172607,
+      "rewards/rejected": -0.011700518429279327,
+      "step": 272
+    },
+    {
+      "epoch": 17.52,
+      "grad_norm": 4.392764568328857,
+      "learning_rate": 4.897716934628217e-07,
+      "logits/chosen": -2.6796340942382812,
+      "logits/rejected": -2.6666793823242188,
+      "logps/chosen": -122.01671600341797,
+      "logps/rejected": -97.74385070800781,
+      "loss": 0.3047,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.0594760179519653,
+      "rewards/margins": 1.1904582977294922,
+      "rewards/rejected": -0.1309822052717209,
+      "step": 276
+    },
+    {
+      "epoch": 17.78,
+      "grad_norm": 4.647542953491211,
+      "learning_rate": 4.894493084499736e-07,
+      "logits/chosen": -2.7203075885772705,
+      "logits/rejected": -2.7299890518188477,
+      "logps/chosen": -120.08210754394531,
+      "logps/rejected": -98.8072738647461,
+      "loss": 0.3496,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 1.0658477544784546,
+      "rewards/margins": 1.065575122833252,
+      "rewards/rejected": 0.0002726661041378975,
+      "step": 280
+    },
+    {
+      "epoch": 17.78,
+      "eval_logits/chosen": -2.798058271408081,
+      "eval_logits/rejected": -2.8771989345550537,
+      "eval_logps/chosen": -124.81001281738281,
+      "eval_logps/rejected": -96.83968353271484,
+      "eval_loss": 0.44096532464027405,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 0.9259332418441772,
+      "eval_rewards/margins": 0.799013078212738,
+      "eval_rewards/rejected": 0.12692023813724518,
+      "eval_runtime": 19.3925,
+      "eval_samples_per_second": 1.444,
+      "eval_steps_per_second": 1.444,
+      "step": 280
+    },
+    {
+      "epoch": 18.03,
+      "grad_norm": 4.158952236175537,
+      "learning_rate": 4.891220308002547e-07,
+      "logits/chosen": -2.6795365810394287,
+      "logits/rejected": -2.7270519733428955,
+      "logps/chosen": -120.39765167236328,
+      "logps/rejected": -100.02141571044922,
+      "loss": 0.319,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 1.0584629774093628,
+      "rewards/margins": 1.19419264793396,
+      "rewards/rejected": -0.13572952151298523,
+      "step": 284
+    },
+    {
+      "epoch": 18.29,
+      "grad_norm": 4.605132579803467,
+      "learning_rate": 4.887898672008871e-07,
+      "logits/chosen": -2.696953296661377,
+      "logits/rejected": -2.718316078186035,
+      "logps/chosen": -120.32958984375,
+      "logps/rejected": -101.76454162597656,
+      "loss": 0.331,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 1.0935906171798706,
+      "rewards/margins": 1.1072776317596436,
+      "rewards/rejected": -0.013687020167708397,
+      "step": 288
+    },
+    {
+      "epoch": 18.54,
+      "grad_norm": 5.824638843536377,
+      "learning_rate": 4.884528244389268e-07,
+      "logits/chosen": -2.6889617443084717,
+      "logits/rejected": -2.690692663192749,
+      "logps/chosen": -120.37040710449219,
+      "logps/rejected": -98.27632141113281,
+      "loss": 0.3204,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 1.1049582958221436,
+      "rewards/margins": 1.175376057624817,
+      "rewards/rejected": -0.07041767239570618,
+      "step": 292
+    },
+    {
+      "epoch": 18.79,
+      "grad_norm": 5.431331634521484,
+      "learning_rate": 4.881109094011253e-07,
+      "logits/chosen": -2.740906238555908,
+      "logits/rejected": -2.725914478302002,
+      "logps/chosen": -124.35327911376953,
+      "logps/rejected": -100.71537780761719,
+      "loss": 0.2884,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.2057325839996338,
+      "rewards/margins": 1.3113868236541748,
+      "rewards/rejected": -0.10565409809350967,
+      "step": 296
+    },
+    {
+      "epoch": 19.05,
+      "grad_norm": 4.33558988571167,
+      "learning_rate": 4.877641290737883e-07,
+      "logits/chosen": -2.700420379638672,
+      "logits/rejected": -2.7527971267700195,
+      "logps/chosen": -117.01264953613281,
+      "logps/rejected": -101.50834655761719,
+      "loss": 0.2757,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.150115728378296,
+      "rewards/margins": 1.34402596950531,
+      "rewards/rejected": -0.19391010701656342,
+      "step": 300
+    },
+    {
+      "epoch": 19.3,
+      "grad_norm": 4.987298488616943,
+      "learning_rate": 4.874124905426338e-07,
+      "logits/chosen": -2.715477705001831,
+      "logits/rejected": -2.760146141052246,
+      "logps/chosen": -117.20336151123047,
+      "logps/rejected": -94.89369201660156,
+      "loss": 0.3244,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 1.107708215713501,
+      "rewards/margins": 1.1569105386734009,
+      "rewards/rejected": -0.04920227825641632,
+      "step": 304
+    },
+    {
+      "epoch": 19.56,
+      "grad_norm": 4.090747833251953,
+      "learning_rate": 4.870560009926464e-07,
+      "logits/chosen": -2.6785225868225098,
+      "logits/rejected": -2.6984167098999023,
+      "logps/chosen": -112.27914428710938,
+      "logps/rejected": -94.59768676757812,
+      "loss": 0.2833,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.2028326988220215,
+      "rewards/margins": 1.334948182106018,
+      "rewards/rejected": -0.13211557269096375,
+      "step": 308
+    },
+    {
+      "epoch": 19.81,
+      "grad_norm": 5.0767083168029785,
+      "learning_rate": 4.866946677079314e-07,
+      "logits/chosen": -2.712416410446167,
+      "logits/rejected": -2.7296035289764404,
+      "logps/chosen": -121.65861511230469,
+      "logps/rejected": -106.11611938476562,
+      "loss": 0.2893,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.1842504739761353,
+      "rewards/margins": 1.3191744089126587,
+      "rewards/rejected": -0.13492396473884583,
+      "step": 312
+    },
+    {
+      "epoch": 20.06,
+      "grad_norm": 4.1403326988220215,
+      "learning_rate": 4.863284980715648e-07,
+      "logits/chosen": -2.658827781677246,
+      "logits/rejected": -2.631199836730957,
+      "logps/chosen": -129.3588409423828,
+      "logps/rejected": -103.25465393066406,
+      "loss": 0.2678,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.291969895362854,
+      "rewards/margins": 1.4036282300949097,
+      "rewards/rejected": -0.11165820807218552,
+      "step": 316
+    },
+    {
+      "epoch": 20.32,
+      "grad_norm": 4.350497245788574,
+      "learning_rate": 4.859574995654441e-07,
+      "logits/chosen": -2.761063814163208,
+      "logits/rejected": -2.7751922607421875,
+      "logps/chosen": -114.76087951660156,
+      "logps/rejected": -100.04124450683594,
+      "loss": 0.298,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.1642746925354004,
+      "rewards/margins": 1.3020979166030884,
+      "rewards/rejected": -0.13782335817813873,
+      "step": 320
+    },
+    {
+      "epoch": 20.32,
+      "eval_logits/chosen": -2.792503595352173,
+      "eval_logits/rejected": -2.870570421218872,
+      "eval_logps/chosen": -123.9360122680664,
+      "eval_logps/rejected": -96.95860290527344,
+      "eval_loss": 0.4256473481655121,
+      "eval_rewards/accuracies": 0.8214285969734192,
+      "eval_rewards/chosen": 1.0133329629898071,
+      "eval_rewards/margins": 0.8983038663864136,
+      "eval_rewards/rejected": 0.11502902954816818,
+      "eval_runtime": 19.3701,
+      "eval_samples_per_second": 1.446,
+      "eval_steps_per_second": 1.446,
+      "step": 320
+    },
+    {
+      "epoch": 20.57,
+      "grad_norm": 4.334249973297119,
+      "learning_rate": 4.855816797701336e-07,
+      "logits/chosen": -2.690117359161377,
+      "logits/rejected": -2.7377476692199707,
+      "logps/chosen": -116.01565551757812,
+      "logps/rejected": -102.1231689453125,
+      "loss": 0.2872,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 1.2394335269927979,
+      "rewards/margins": 1.3586478233337402,
+      "rewards/rejected": -0.11921422928571701,
+      "step": 324
+    },
+    {
+      "epoch": 20.83,
+      "grad_norm": 4.597221851348877,
+      "learning_rate": 4.85201046364711e-07,
+      "logits/chosen": -2.741600751876831,
+      "logits/rejected": -2.712954044342041,
+      "logps/chosen": -124.32752990722656,
+      "logps/rejected": -101.2073974609375,
+      "loss": 0.2676,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.22231924533844,
+      "rewards/margins": 1.4293614625930786,
+      "rewards/rejected": -0.20704227685928345,
+      "step": 328
+    },
+    {
+      "epoch": 21.08,
+      "grad_norm": 3.682258367538452,
+      "learning_rate": 4.848156071266095e-07,
+      "logits/chosen": -2.6585566997528076,
+      "logits/rejected": -2.704587459564209,
+      "logps/chosen": -120.91087341308594,
+      "logps/rejected": -98.83493041992188,
+      "loss": 0.2705,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.2277172803878784,
+      "rewards/margins": 1.3790152072906494,
+      "rewards/rejected": -0.15129785239696503,
+      "step": 332
+    },
+    {
+      "epoch": 21.33,
+      "grad_norm": 3.7775039672851562,
+      "learning_rate": 4.844253699314596e-07,
+      "logits/chosen": -2.7104837894439697,
+      "logits/rejected": -2.7378451824188232,
+      "logps/chosen": -117.22598266601562,
+      "logps/rejected": -97.59563446044922,
+      "loss": 0.2755,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.2548604011535645,
+      "rewards/margins": 1.3694878816604614,
+      "rewards/rejected": -0.11462761461734772,
+      "step": 336
+    },
+    {
+      "epoch": 21.59,
+      "grad_norm": 4.84611701965332,
+      "learning_rate": 4.840303427529273e-07,
+      "logits/chosen": -2.6837968826293945,
+      "logits/rejected": -2.7071595191955566,
+      "logps/chosen": -121.88581085205078,
+      "logps/rejected": -102.45454406738281,
+      "loss": 0.286,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 1.2159500122070312,
+      "rewards/margins": 1.3315218687057495,
+      "rewards/rejected": -0.11557181179523468,
+      "step": 340
+    },
+    {
+      "epoch": 21.84,
+      "grad_norm": 3.760068655014038,
+      "learning_rate": 4.836305336625523e-07,
+      "logits/chosen": -2.709066390991211,
+      "logits/rejected": -2.73175048828125,
+      "logps/chosen": -117.1901626586914,
+      "logps/rejected": -105.62377166748047,
+      "loss": 0.2661,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.3225661516189575,
+      "rewards/margins": 1.445369005203247,
+      "rewards/rejected": -0.12280277907848358,
+      "step": 344
+    },
+    {
+      "epoch": 22.1,
+      "grad_norm": 3.8488616943359375,
+      "learning_rate": 4.832259508295822e-07,
+      "logits/chosen": -2.7598538398742676,
+      "logits/rejected": -2.776313066482544,
+      "logps/chosen": -117.15644836425781,
+      "logps/rejected": -93.95286560058594,
+      "loss": 0.2711,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 1.3003535270690918,
+      "rewards/margins": 1.5021978616714478,
+      "rewards/rejected": -0.20184442400932312,
+      "step": 348
+    },
+    {
+      "epoch": 22.35,
+      "grad_norm": 3.152104377746582,
+      "learning_rate": 4.828166025208058e-07,
+      "logits/chosen": -2.6503114700317383,
+      "logits/rejected": -2.6512064933776855,
+      "logps/chosen": -118.29179382324219,
+      "logps/rejected": -97.83879852294922,
+      "loss": 0.2394,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.2750773429870605,
+      "rewards/margins": 1.585729718208313,
+      "rewards/rejected": -0.31065240502357483,
+      "step": 352
+    },
+    {
+      "epoch": 22.6,
+      "grad_norm": 4.1737165451049805,
+      "learning_rate": 4.824024971003845e-07,
+      "logits/chosen": -2.708979845046997,
+      "logits/rejected": -2.718264579772949,
+      "logps/chosen": -113.75922393798828,
+      "logps/rejected": -100.5313720703125,
+      "loss": 0.2626,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.3227206468582153,
+      "rewards/margins": 1.4637939929962158,
+      "rewards/rejected": -0.1410733461380005,
+      "step": 356
+    },
+    {
+      "epoch": 22.86,
+      "grad_norm": 4.2186737060546875,
+      "learning_rate": 4.819836430296808e-07,
+      "logits/chosen": -2.688643217086792,
+      "logits/rejected": -2.7126376628875732,
+      "logps/chosen": -119.3924560546875,
+      "logps/rejected": -102.88687896728516,
+      "loss": 0.272,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.3319997787475586,
+      "rewards/margins": 1.3775808811187744,
+      "rewards/rejected": -0.04558102786540985,
+      "step": 360
+    },
+    {
+      "epoch": 22.86,
+      "eval_logits/chosen": -2.7863781452178955,
+      "eval_logits/rejected": -2.8632652759552,
+      "eval_logps/chosen": -123.22322082519531,
+      "eval_logps/rejected": -97.17105865478516,
+      "eval_loss": 0.4147447645664215,
+      "eval_rewards/accuracies": 0.8214285969734192,
+      "eval_rewards/chosen": 1.0846121311187744,
+      "eval_rewards/margins": 0.9908290505409241,
+      "eval_rewards/rejected": 0.09378316253423691,
+      "eval_runtime": 19.3953,
+      "eval_samples_per_second": 1.444,
+      "eval_steps_per_second": 1.444,
+      "step": 360
+    },
+    {
+      "epoch": 23.11,
+      "grad_norm": 3.8801398277282715,
+      "learning_rate": 4.815600488670862e-07,
+      "logits/chosen": -2.661937713623047,
+      "logits/rejected": -2.69406795501709,
+      "logps/chosen": -120.42941284179688,
+      "logps/rejected": -107.56249237060547,
+      "loss": 0.2463,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.303568959236145,
+      "rewards/margins": 1.5796161890029907,
+      "rewards/rejected": -0.2760472595691681,
+      "step": 364
+    },
+    {
+      "epoch": 23.37,
+      "grad_norm": 3.6953530311584473,
+      "learning_rate": 4.811317232678456e-07,
+      "logits/chosen": -2.696389675140381,
+      "logits/rejected": -2.7162017822265625,
+      "logps/chosen": -115.95695495605469,
+      "logps/rejected": -95.26593017578125,
+      "loss": 0.2557,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.3928656578063965,
+      "rewards/margins": 1.5144150257110596,
+      "rewards/rejected": -0.12154923379421234,
+      "step": 368
+    },
+    {
+      "epoch": 23.62,
+      "grad_norm": 4.2311553955078125,
+      "learning_rate": 4.806986749838806e-07,
+      "logits/chosen": -2.735870599746704,
+      "logits/rejected": -2.7359132766723633,
+      "logps/chosen": -120.84663391113281,
+      "logps/rejected": -107.17831420898438,
+      "loss": 0.2922,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.2136850357055664,
+      "rewards/margins": 1.3251327276229858,
+      "rewards/rejected": -0.11144763976335526,
+      "step": 372
+    },
+    {
+      "epoch": 23.87,
+      "grad_norm": 3.250326633453369,
+      "learning_rate": 4.802609128636112e-07,
+      "logits/chosen": -2.6809797286987305,
+      "logits/rejected": -2.700901508331299,
+      "logps/chosen": -121.92070007324219,
+      "logps/rejected": -99.91931915283203,
+      "loss": 0.2062,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.4374094009399414,
+      "rewards/margins": 1.8276853561401367,
+      "rewards/rejected": -0.3902759850025177,
+      "step": 376
+    },
+    {
+      "epoch": 24.13,
+      "grad_norm": 4.59958553314209,
+      "learning_rate": 4.798184458517744e-07,
+      "logits/chosen": -2.7332396507263184,
+      "logits/rejected": -2.768554210662842,
+      "logps/chosen": -114.1657943725586,
+      "logps/rejected": -100.7848892211914,
+      "loss": 0.2353,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.4016469717025757,
+      "rewards/margins": 1.5988764762878418,
+      "rewards/rejected": -0.19722968339920044,
+      "step": 380
+    },
+    {
+      "epoch": 24.38,
+      "grad_norm": 3.6598093509674072,
+      "learning_rate": 4.793712829892414e-07,
+      "logits/chosen": -2.7024924755096436,
+      "logits/rejected": -2.7085440158843994,
+      "logps/chosen": -114.50930786132812,
+      "logps/rejected": -104.57618713378906,
+      "loss": 0.2538,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.4238942861557007,
+      "rewards/margins": 1.5259473323822021,
+      "rewards/rejected": -0.10205309092998505,
+      "step": 384
+    },
+    {
+      "epoch": 24.63,
+      "grad_norm": 3.3857998847961426,
+      "learning_rate": 4.789194334128338e-07,
+      "logits/chosen": -2.7139151096343994,
+      "logits/rejected": -2.7087197303771973,
+      "logps/chosen": -121.57815551757812,
+      "logps/rejected": -102.66044616699219,
+      "loss": 0.2324,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.3619202375411987,
+      "rewards/margins": 1.667473316192627,
+      "rewards/rejected": -0.305553138256073,
+      "step": 388
+    },
+    {
+      "epoch": 24.89,
+      "grad_norm": 4.038039207458496,
+      "learning_rate": 4.784629063551354e-07,
+      "logits/chosen": -2.651282548904419,
+      "logits/rejected": -2.679689884185791,
+      "logps/chosen": -118.9875717163086,
+      "logps/rejected": -97.62678527832031,
+      "loss": 0.2304,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 1.3316906690597534,
+      "rewards/margins": 1.7062404155731201,
+      "rewards/rejected": -0.3745496869087219,
+      "step": 392
+    },
+    {
+      "epoch": 25.14,
+      "grad_norm": 3.0911149978637695,
+      "learning_rate": 4.780017111443047e-07,
+      "logits/chosen": -2.718074083328247,
+      "logits/rejected": -2.7447762489318848,
+      "logps/chosen": -121.20558166503906,
+      "logps/rejected": -105.63926696777344,
+      "loss": 0.2411,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.4981815814971924,
+      "rewards/margins": 1.644660472869873,
+      "rewards/rejected": -0.14647886157035828,
+      "step": 396
+    },
+    {
+      "epoch": 25.4,
+      "grad_norm": 3.236288070678711,
+      "learning_rate": 4.775358572038845e-07,
+      "logits/chosen": -2.693341016769409,
+      "logits/rejected": -2.681511878967285,
+      "logps/chosen": -114.28794860839844,
+      "logps/rejected": -91.69850158691406,
+      "loss": 0.259,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.3284053802490234,
+      "rewards/margins": 1.5037521123886108,
+      "rewards/rejected": -0.17534679174423218,
+      "step": 400
+    },
+    {
+      "epoch": 25.4,
+      "eval_logits/chosen": -2.7796428203582764,
+      "eval_logits/rejected": -2.8552372455596924,
+      "eval_logps/chosen": -122.60997009277344,
+      "eval_logps/rejected": -97.46685028076172,
+      "eval_loss": 0.4054030478000641,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.1459368467330933,
+      "eval_rewards/margins": 1.0817322731018066,
+      "eval_rewards/rejected": 0.06420455127954483,
+      "eval_runtime": 19.3983,
+      "eval_samples_per_second": 1.443,
+      "eval_steps_per_second": 1.443,
+      "step": 400
+    },
+    {
+      "epoch": 25.65,
+      "grad_norm": 3.2894155979156494,
+      "learning_rate": 4.770653540526078e-07,
+      "logits/chosen": -2.652012825012207,
+      "logits/rejected": -2.690704345703125,
+      "logps/chosen": -118.52864837646484,
+      "logps/rejected": -102.0433578491211,
+      "loss": 0.1918,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.5184407234191895,
+      "rewards/margins": 1.8730953931808472,
+      "rewards/rejected": -0.3546547293663025,
+      "step": 404
+    },
+    {
+      "epoch": 25.9,
+      "grad_norm": 4.958095550537109,
+      "learning_rate": 4.765902113042053e-07,
+      "logits/chosen": -2.7191715240478516,
+      "logits/rejected": -2.701702833175659,
+      "logps/chosen": -115.93360900878906,
+      "logps/rejected": -100.49573516845703,
+      "loss": 0.2536,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.357507348060608,
+      "rewards/margins": 1.542129635810852,
+      "rewards/rejected": -0.18462230265140533,
+      "step": 408
+    },
+    {
+      "epoch": 26.16,
+      "grad_norm": 3.1129214763641357,
+      "learning_rate": 4.7611043866720737e-07,
+      "logits/chosen": -2.6934893131256104,
+      "logits/rejected": -2.7433109283447266,
+      "logps/chosen": -117.78741455078125,
+      "logps/rejected": -102.46746063232422,
+      "loss": 0.2137,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 1.484704613685608,
+      "rewards/margins": 1.8401193618774414,
+      "rewards/rejected": -0.3554147481918335,
+      "step": 412
+    },
+    {
+      "epoch": 26.41,
+      "grad_norm": 3.2823047637939453,
+      "learning_rate": 4.7562604594474643e-07,
+      "logits/chosen": -2.6928577423095703,
+      "logits/rejected": -2.708937644958496,
+      "logps/chosen": -124.94667053222656,
+      "logps/rejected": -106.61329650878906,
+      "loss": 0.2137,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.471519112586975,
+      "rewards/margins": 1.7852158546447754,
+      "rewards/rejected": -0.31369656324386597,
+      "step": 416
+    },
+    {
+      "epoch": 26.67,
+      "grad_norm": 3.7071120738983154,
+      "learning_rate": 4.7513704303435674e-07,
+      "logits/chosen": -2.670248031616211,
+      "logits/rejected": -2.6764914989471436,
+      "logps/chosen": -110.71532440185547,
+      "logps/rejected": -101.84510803222656,
+      "loss": 0.229,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.4525586366653442,
+      "rewards/margins": 1.6690635681152344,
+      "rewards/rejected": -0.21650481224060059,
+      "step": 420
+    },
+    {
+      "epoch": 26.92,
+      "grad_norm": 3.449319362640381,
+      "learning_rate": 4.7464343992777165e-07,
+      "logits/chosen": -2.688829183578491,
+      "logits/rejected": -2.7286489009857178,
+      "logps/chosen": -115.69140625,
+      "logps/rejected": -103.97221374511719,
+      "loss": 0.2231,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.4910831451416016,
+      "rewards/margins": 1.7472798824310303,
+      "rewards/rejected": -0.2561967670917511,
+      "step": 424
+    },
+    {
+      "epoch": 27.17,
+      "grad_norm": 2.4958574771881104,
+      "learning_rate": 4.741452467107199e-07,
+      "logits/chosen": -2.6915664672851562,
+      "logits/rejected": -2.7021422386169434,
+      "logps/chosen": -110.91499328613281,
+      "logps/rejected": -98.70269775390625,
+      "loss": 0.233,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.4354866743087769,
+      "rewards/margins": 1.7168078422546387,
+      "rewards/rejected": -0.28132107853889465,
+      "step": 428
+    },
+    {
+      "epoch": 27.43,
+      "grad_norm": 3.7154319286346436,
+      "learning_rate": 4.7364247356271927e-07,
+      "logits/chosen": -2.6695199012756348,
+      "logits/rejected": -2.6698789596557617,
+      "logps/chosen": -118.46913146972656,
+      "logps/rejected": -102.86263275146484,
+      "loss": 0.2337,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.4410640001296997,
+      "rewards/margins": 1.6770535707473755,
+      "rewards/rejected": -0.23598948121070862,
+      "step": 432
+    },
+    {
+      "epoch": 27.68,
+      "grad_norm": 3.101527690887451,
+      "learning_rate": 4.731351307568687e-07,
+      "logits/chosen": -2.6910083293914795,
+      "logits/rejected": -2.6801414489746094,
+      "logps/chosen": -115.44275665283203,
+      "logps/rejected": -97.7877197265625,
+      "loss": 0.2,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.5193870067596436,
+      "rewards/margins": 1.8750090599060059,
+      "rewards/rejected": -0.35562214255332947,
+      "step": 436
+    },
+    {
+      "epoch": 27.94,
+      "grad_norm": 3.9017672538757324,
+      "learning_rate": 4.7262322865963846e-07,
+      "logits/chosen": -2.7122743129730225,
+      "logits/rejected": -2.723294258117676,
+      "logps/chosen": -116.90724182128906,
+      "logps/rejected": -104.45236206054688,
+      "loss": 0.1959,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.5187357664108276,
+      "rewards/margins": 1.918578863143921,
+      "rewards/rejected": -0.39984312653541565,
+      "step": 440
+    },
+    {
+      "epoch": 27.94,
+      "eval_logits/chosen": -2.7724335193634033,
+      "eval_logits/rejected": -2.8468399047851562,
+      "eval_logps/chosen": -122.11509704589844,
+      "eval_logps/rejected": -97.87715148925781,
+      "eval_loss": 0.39711207151412964,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.1954243183135986,
+      "eval_rewards/margins": 1.1722500324249268,
+      "eval_rewards/rejected": 0.023174213245511055,
+      "eval_runtime": 19.4446,
+      "eval_samples_per_second": 1.44,
+      "eval_steps_per_second": 1.44,
+      "step": 440
+    },
+    {
+      "epoch": 28.19,
+      "grad_norm": 4.0338592529296875,
+      "learning_rate": 4.721067777306581e-07,
+      "logits/chosen": -2.6722593307495117,
+      "logits/rejected": -2.682727575302124,
+      "logps/chosen": -119.6982650756836,
+      "logps/rejected": -105.52352905273438,
+      "loss": 0.2108,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.5048611164093018,
+      "rewards/margins": 1.8753631114959717,
+      "rewards/rejected": -0.3705020546913147,
+      "step": 444
+    },
+    {
+      "epoch": 28.44,
+      "grad_norm": 3.167048692703247,
+      "learning_rate": 4.71585788522503e-07,
+      "logits/chosen": -2.652644157409668,
+      "logits/rejected": -2.6818065643310547,
+      "logps/chosen": -113.42478942871094,
+      "logps/rejected": -104.37007141113281,
+      "loss": 0.2076,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.474198341369629,
+      "rewards/margins": 1.7973999977111816,
+      "rewards/rejected": -0.32320165634155273,
+      "step": 448
+    },
+    {
+      "epoch": 28.7,
+      "grad_norm": 3.4248368740081787,
+      "learning_rate": 4.7106027168047833e-07,
+      "logits/chosen": -2.7117884159088135,
+      "logits/rejected": -2.77234148979187,
+      "logps/chosen": -116.26769256591797,
+      "logps/rejected": -101.22008514404297,
+      "loss": 0.1858,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.5492609739303589,
+      "rewards/margins": 1.9960644245147705,
+      "rewards/rejected": -0.4468035101890564,
+      "step": 452
+    },
+    {
+      "epoch": 28.95,
+      "grad_norm": 2.855328321456909,
+      "learning_rate": 4.705302379424022e-07,
+      "logits/chosen": -2.6717069149017334,
+      "logits/rejected": -2.672755718231201,
+      "logps/chosen": -116.75204467773438,
+      "logps/rejected": -106.22608184814453,
+      "loss": 0.2065,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.5143406391143799,
+      "rewards/margins": 1.8520642518997192,
+      "rewards/rejected": -0.33772367238998413,
+      "step": 456
+    },
+    {
+      "epoch": 29.21,
+      "grad_norm": 2.8747594356536865,
+      "learning_rate": 4.699956981383857e-07,
+      "logits/chosen": -2.693591356277466,
+      "logits/rejected": -2.662400960922241,
+      "logps/chosen": -120.1160888671875,
+      "logps/rejected": -91.72486877441406,
+      "loss": 0.1874,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.6478428840637207,
+      "rewards/margins": 2.0628483295440674,
+      "rewards/rejected": -0.4150054454803467,
+      "step": 460
+    },
+    {
+      "epoch": 29.46,
+      "grad_norm": 2.8170723915100098,
+      "learning_rate": 4.694566631906116e-07,
+      "logits/chosen": -2.6773173809051514,
+      "logits/rejected": -2.672825574874878,
+      "logps/chosen": -113.59075927734375,
+      "logps/rejected": -103.25709533691406,
+      "loss": 0.1984,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.4863029718399048,
+      "rewards/margins": 1.892244577407837,
+      "rewards/rejected": -0.405941367149353,
+      "step": 464
+    },
+    {
+      "epoch": 29.71,
+      "grad_norm": 3.132080078125,
+      "learning_rate": 4.689131441131118e-07,
+      "logits/chosen": -2.6520447731018066,
+      "logits/rejected": -2.6872832775115967,
+      "logps/chosen": -118.07131958007812,
+      "logps/rejected": -107.88346862792969,
+      "loss": 0.2029,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.6008590459823608,
+      "rewards/margins": 1.909461259841919,
+      "rewards/rejected": -0.30860215425491333,
+      "step": 468
+    },
+    {
+      "epoch": 29.97,
+      "grad_norm": 3.000974416732788,
+      "learning_rate": 4.6836515201154135e-07,
+      "logits/chosen": -2.709559202194214,
+      "logits/rejected": -2.723712205886841,
+      "logps/chosen": -112.09906768798828,
+      "logps/rejected": -106.09660339355469,
+      "loss": 0.2037,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.5191339254379272,
+      "rewards/margins": 1.8524271249771118,
+      "rewards/rejected": -0.3332933187484741,
+      "step": 472
+    },
+    {
+      "epoch": 30.22,
+      "grad_norm": 2.544147491455078,
+      "learning_rate": 4.678126980829524e-07,
+      "logits/chosen": -2.678325653076172,
+      "logits/rejected": -2.699833631515503,
+      "logps/chosen": -115.4670639038086,
+      "logps/rejected": -101.5027847290039,
+      "loss": 0.1813,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.4956222772598267,
+      "rewards/margins": 1.9856685400009155,
+      "rewards/rejected": -0.4900462329387665,
+      "step": 476
+    },
+    {
+      "epoch": 30.48,
+      "grad_norm": 2.9770796298980713,
+      "learning_rate": 4.672557936155649e-07,
+      "logits/chosen": -2.708721160888672,
+      "logits/rejected": -2.728306770324707,
+      "logps/chosen": -109.71602630615234,
+      "logps/rejected": -98.1273422241211,
+      "loss": 0.1946,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.5404164791107178,
+      "rewards/margins": 1.9511140584945679,
+      "rewards/rejected": -0.4106977581977844,
+      "step": 480
+    },
+    {
+      "epoch": 30.48,
+      "eval_logits/chosen": -2.764019012451172,
+      "eval_logits/rejected": -2.8370797634124756,
+      "eval_logps/chosen": -121.74720001220703,
+      "eval_logps/rejected": -98.30638122558594,
+      "eval_loss": 0.393496572971344,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.232213020324707,
+      "eval_rewards/margins": 1.2519627809524536,
+      "eval_rewards/rejected": -0.019749613478779793,
+      "eval_runtime": 19.4183,
+      "eval_samples_per_second": 1.442,
+      "eval_steps_per_second": 1.442,
+      "step": 480
+    },
+    {
+      "epoch": 30.73,
+      "grad_norm": 3.826716661453247,
+      "learning_rate": 4.6669444998853605e-07,
+      "logits/chosen": -2.6732892990112305,
+      "logits/rejected": -2.700220823287964,
+      "logps/chosen": -117.65553283691406,
+      "logps/rejected": -111.19263458251953,
+      "loss": 0.1876,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.6948100328445435,
+      "rewards/margins": 2.0582737922668457,
+      "rewards/rejected": -0.3634639382362366,
+      "step": 484
+    },
+    {
+      "epoch": 30.98,
+      "grad_norm": 3.326704740524292,
+      "learning_rate": 4.661286786717278e-07,
+      "logits/chosen": -2.6515111923217773,
+      "logits/rejected": -2.6504664421081543,
+      "logps/chosen": -120.06623840332031,
+      "logps/rejected": -101.47470092773438,
+      "loss": 0.1828,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.5970648527145386,
+      "rewards/margins": 2.0543787479400635,
+      "rewards/rejected": -0.45731377601623535,
+      "step": 488
+    },
+    {
+      "epoch": 31.24,
+      "grad_norm": 2.700209856033325,
+      "learning_rate": 4.6555849122547263e-07,
+      "logits/chosen": -2.6469173431396484,
+      "logits/rejected": -2.666424036026001,
+      "logps/chosen": -111.76505279541016,
+      "logps/rejected": -103.40081024169922,
+      "loss": 0.185,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.524863839149475,
+      "rewards/margins": 1.9786373376846313,
+      "rewards/rejected": -0.4537736773490906,
+      "step": 492
+    },
+    {
+      "epoch": 31.49,
+      "grad_norm": 2.592606544494629,
+      "learning_rate": 4.649838993003372e-07,
+      "logits/chosen": -2.7192447185516357,
+      "logits/rejected": -2.726841926574707,
+      "logps/chosen": -112.72746276855469,
+      "logps/rejected": -100.51376342773438,
+      "loss": 0.1678,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.5847443342208862,
+      "rewards/margins": 2.0699594020843506,
+      "rewards/rejected": -0.4852150082588196,
+      "step": 496
+    },
+    {
+      "epoch": 31.75,
+      "grad_norm": 2.768076181411743,
+      "learning_rate": 4.644049146368843e-07,
+      "logits/chosen": -2.6624693870544434,
+      "logits/rejected": -2.677476406097412,
+      "logps/chosen": -119.13687896728516,
+      "logps/rejected": -103.49778747558594,
+      "loss": 0.1873,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7677032947540283,
+      "rewards/margins": 2.089017868041992,
+      "rewards/rejected": -0.32131490111351013,
+      "step": 500
+    },
+    {
+      "epoch": 32.0,
+      "grad_norm": 2.4647607803344727,
+      "learning_rate": 4.638215490654329e-07,
+      "logits/chosen": -2.6946330070495605,
+      "logits/rejected": -2.6913187503814697,
+      "logps/chosen": -118.64775085449219,
+      "logps/rejected": -107.40900421142578,
+      "loss": 0.1847,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.5691322088241577,
+      "rewards/margins": 2.105942964553833,
+      "rewards/rejected": -0.5368106365203857,
+      "step": 504
+    },
+    {
+      "epoch": 32.25,
+      "grad_norm": 3.723209857940674,
+      "learning_rate": 4.6323381450581665e-07,
+      "logits/chosen": -2.6984715461730957,
+      "logits/rejected": -2.686072826385498,
+      "logps/chosen": -118.19576263427734,
+      "logps/rejected": -104.87760162353516,
+      "loss": 0.1853,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.504220724105835,
+      "rewards/margins": 2.023249864578247,
+      "rewards/rejected": -0.519028902053833,
+      "step": 508
+    },
+    {
+      "epoch": 32.51,
+      "grad_norm": 2.6810081005096436,
+      "learning_rate": 4.6264172296714e-07,
+      "logits/chosen": -2.6358213424682617,
+      "logits/rejected": -2.6452157497406006,
+      "logps/chosen": -116.47501373291016,
+      "logps/rejected": -104.9029769897461,
+      "loss": 0.1948,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.6027023792266846,
+      "rewards/margins": 2.105417251586914,
+      "rewards/rejected": -0.5027150511741638,
+      "step": 512
+    },
+    {
+      "epoch": 32.76,
+      "grad_norm": 2.1455435752868652,
+      "learning_rate": 4.6204528654753304e-07,
+      "logits/chosen": -2.692194700241089,
+      "logits/rejected": -2.705289125442505,
+      "logps/chosen": -113.38731384277344,
+      "logps/rejected": -103.49237823486328,
+      "loss": 0.1617,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7136329412460327,
+      "rewards/margins": 2.1900761127471924,
+      "rewards/rejected": -0.47644317150115967,
+      "step": 516
+    },
+    {
+      "epoch": 33.02,
+      "grad_norm": 3.499128818511963,
+      "learning_rate": 4.614445174339045e-07,
+      "logits/chosen": -2.661409616470337,
+      "logits/rejected": -2.700990915298462,
+      "logps/chosen": -113.71514892578125,
+      "logps/rejected": -103.91769409179688,
+      "loss": 0.1507,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7427312135696411,
+      "rewards/margins": 2.1958351135253906,
+      "rewards/rejected": -0.45310381054878235,
+      "step": 520
+    },
+    {
+      "epoch": 33.02,
+      "eval_logits/chosen": -2.755352020263672,
+      "eval_logits/rejected": -2.826873540878296,
+      "eval_logps/chosen": -121.47989654541016,
+      "eval_logps/rejected": -98.8564453125,
+      "eval_loss": 0.3899853527545929,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.2589442729949951,
+      "eval_rewards/margins": 1.333700180053711,
+      "eval_rewards/rejected": -0.07475597411394119,
+      "eval_runtime": 19.3899,
+      "eval_samples_per_second": 1.444,
+      "eval_steps_per_second": 1.444,
+      "step": 520
+    },
+    {
+      "epoch": 33.27,
+      "grad_norm": 2.419506549835205,
+      "learning_rate": 4.6083942790169206e-07,
+      "logits/chosen": -2.6735386848449707,
+      "logits/rejected": -2.696384906768799,
+      "logps/chosen": -111.79097747802734,
+      "logps/rejected": -98.7368392944336,
+      "loss": 0.1581,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.5749679803848267,
+      "rewards/margins": 2.265718460083008,
+      "rewards/rejected": -0.690750777721405,
+      "step": 524
+    },
+    {
+      "epoch": 33.52,
+      "grad_norm": 2.1046078205108643,
+      "learning_rate": 4.6023003031461227e-07,
+      "logits/chosen": -2.6465351581573486,
+      "logits/rejected": -2.6622884273529053,
+      "logps/chosen": -114.73282623291016,
+      "logps/rejected": -103.12973022460938,
+      "loss": 0.1778,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.6710339784622192,
+      "rewards/margins": 2.119300365447998,
+      "rewards/rejected": -0.4482663571834564,
+      "step": 528
+    },
+    {
+      "epoch": 33.78,
+      "grad_norm": 3.0457003116607666,
+      "learning_rate": 4.596163371244076e-07,
+      "logits/chosen": -2.6422927379608154,
+      "logits/rejected": -2.6462604999542236,
+      "logps/chosen": -114.708740234375,
+      "logps/rejected": -105.07784271240234,
+      "loss": 0.16,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7936890125274658,
+      "rewards/margins": 2.2034826278686523,
+      "rewards/rejected": -0.40979355573654175,
+      "step": 532
+    },
+    {
+      "epoch": 34.03,
+      "grad_norm": 2.8257133960723877,
+      "learning_rate": 4.5899836087059174e-07,
+      "logits/chosen": -2.6918063163757324,
+      "logits/rejected": -2.696005344390869,
+      "logps/chosen": -117.54481506347656,
+      "logps/rejected": -111.74267578125,
+      "loss": 0.1753,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.625847339630127,
+      "rewards/margins": 2.2450764179229736,
+      "rewards/rejected": -0.6192291975021362,
+      "step": 536
+    },
+    {
+      "epoch": 34.29,
+      "grad_norm": 2.5381155014038086,
+      "learning_rate": 4.58376114180194e-07,
+      "logits/chosen": -2.684847116470337,
+      "logits/rejected": -2.6875247955322266,
+      "logps/chosen": -121.1004867553711,
+      "logps/rejected": -101.69570922851562,
+      "loss": 0.1562,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.6668047904968262,
+      "rewards/margins": 2.3632285594940186,
+      "rewards/rejected": -0.6964237689971924,
+      "step": 540
+    },
+    {
+      "epoch": 34.54,
+      "grad_norm": 2.519352912902832,
+      "learning_rate": 4.577496097675009e-07,
+      "logits/chosen": -2.6664843559265137,
+      "logits/rejected": -2.681034803390503,
+      "logps/chosen": -115.94261169433594,
+      "logps/rejected": -108.1679458618164,
+      "loss": 0.1516,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7702218294143677,
+      "rewards/margins": 2.2215099334716797,
+      "rewards/rejected": -0.4512881338596344,
+      "step": 544
+    },
+    {
+      "epoch": 34.79,
+      "grad_norm": 2.789398431777954,
+      "learning_rate": 4.571188604337962e-07,
+      "logits/chosen": -2.668377637863159,
+      "logits/rejected": -2.688797950744629,
+      "logps/chosen": -108.2844467163086,
+      "logps/rejected": -104.94053649902344,
+      "loss": 0.1838,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.559061050415039,
+      "rewards/margins": 2.064011812210083,
+      "rewards/rejected": -0.504950761795044,
+      "step": 548
+    },
+    {
+      "epoch": 35.05,
+      "grad_norm": 3.570829153060913,
+      "learning_rate": 4.5648387906709995e-07,
+      "logits/chosen": -2.6559903621673584,
+      "logits/rejected": -2.650508165359497,
+      "logps/chosen": -112.94985961914062,
+      "logps/rejected": -101.36627197265625,
+      "loss": 0.1484,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.709464192390442,
+      "rewards/margins": 2.303464889526367,
+      "rewards/rejected": -0.5940006375312805,
+      "step": 552
+    },
+    {
+      "epoch": 35.3,
+      "grad_norm": 2.487671136856079,
+      "learning_rate": 4.558446786419045e-07,
+      "logits/chosen": -2.6454193592071533,
+      "logits/rejected": -2.667191982269287,
+      "logps/chosen": -112.91888427734375,
+      "logps/rejected": -100.34318542480469,
+      "loss": 0.1571,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.6041696071624756,
+      "rewards/margins": 2.3108744621276855,
+      "rewards/rejected": -0.7067050337791443,
+      "step": 556
+    },
+    {
+      "epoch": 35.56,
+      "grad_norm": 2.396897315979004,
+      "learning_rate": 4.552012722189099e-07,
+      "logits/chosen": -2.6444239616394043,
+      "logits/rejected": -2.6464180946350098,
+      "logps/chosen": -115.7559814453125,
+      "logps/rejected": -111.63291931152344,
+      "loss": 0.1752,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7378344535827637,
+      "rewards/margins": 2.1783785820007324,
+      "rewards/rejected": -0.4405441880226135,
+      "step": 560
+    },
+    {
+      "epoch": 35.56,
+      "eval_logits/chosen": -2.746135711669922,
+      "eval_logits/rejected": -2.8161184787750244,
+      "eval_logps/chosen": -121.27652740478516,
+      "eval_logps/rejected": -99.45245361328125,
+      "eval_loss": 0.38749733567237854,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.2792812585830688,
+      "eval_rewards/margins": 1.4136368036270142,
+      "eval_rewards/rejected": -0.1343555599451065,
+      "eval_runtime": 19.3831,
+      "eval_samples_per_second": 1.445,
+      "eval_steps_per_second": 1.445,
+      "step": 560
+    },
+    {
+      "epoch": 35.81,
+      "grad_norm": 3.2688581943511963,
+      "learning_rate": 4.5455367294475655e-07,
+      "logits/chosen": -2.7136824131011963,
+      "logits/rejected": -2.724392890930176,
+      "logps/chosen": -114.86489868164062,
+      "logps/rejected": -108.21212768554688,
+      "loss": 0.1485,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7876478433609009,
+      "rewards/margins": 2.3711612224578857,
+      "rewards/rejected": -0.5835133790969849,
+      "step": 564
+    },
+    {
+      "epoch": 36.06,
+      "grad_norm": 2.39674711227417,
+      "learning_rate": 4.5390189405175715e-07,
+      "logits/chosen": -2.6487505435943604,
+      "logits/rejected": -2.651470899581909,
+      "logps/chosen": -111.85699462890625,
+      "logps/rejected": -101.57424926757812,
+      "loss": 0.1222,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7182648181915283,
+      "rewards/margins": 2.550403356552124,
+      "rewards/rejected": -0.8321384787559509,
+      "step": 568
+    },
+    {
+      "epoch": 36.32,
+      "grad_norm": 2.970404624938965,
+      "learning_rate": 4.5324594885762576e-07,
+      "logits/chosen": -2.670048952102661,
+      "logits/rejected": -2.70831036567688,
+      "logps/chosen": -113.16908264160156,
+      "logps/rejected": -116.0752944946289,
+      "loss": 0.1838,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.6036030054092407,
+      "rewards/margins": 2.133262872695923,
+      "rewards/rejected": -0.5296599268913269,
+      "step": 572
+    },
+    {
+      "epoch": 36.57,
+      "grad_norm": 3.4210455417633057,
+      "learning_rate": 4.5258585076520595e-07,
+      "logits/chosen": -2.647081136703491,
+      "logits/rejected": -2.625157594680786,
+      "logps/chosen": -119.30801391601562,
+      "logps/rejected": -101.5343017578125,
+      "loss": 0.1394,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7498310804367065,
+      "rewards/margins": 2.5768487453460693,
+      "rewards/rejected": -0.8270176649093628,
+      "step": 576
+    },
+    {
+      "epoch": 36.83,
+      "grad_norm": 2.8785665035247803,
+      "learning_rate": 4.5192161326219716e-07,
+      "logits/chosen": -2.678386688232422,
+      "logits/rejected": -2.6918272972106934,
+      "logps/chosen": -111.96650695800781,
+      "logps/rejected": -100.6183853149414,
+      "loss": 0.1539,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.703883409500122,
+      "rewards/margins": 2.2555694580078125,
+      "rewards/rejected": -0.5516858696937561,
+      "step": 580
+    },
+    {
+      "epoch": 37.08,
+      "grad_norm": 1.953771710395813,
+      "learning_rate": 4.512532499208786e-07,
+      "logits/chosen": -2.622621774673462,
+      "logits/rejected": -2.6352713108062744,
+      "logps/chosen": -117.12508392333984,
+      "logps/rejected": -106.53787231445312,
+      "loss": 0.1165,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.857337474822998,
+      "rewards/margins": 2.5984973907470703,
+      "rewards/rejected": -0.7411599159240723,
+      "step": 584
+    },
+    {
+      "epoch": 37.33,
+      "grad_norm": 1.7565056085586548,
+      "learning_rate": 4.505807743978324e-07,
+      "logits/chosen": -2.6742653846740723,
+      "logits/rejected": -2.6801705360412598,
+      "logps/chosen": -112.03057098388672,
+      "logps/rejected": -106.16900634765625,
+      "loss": 0.145,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8526411056518555,
+      "rewards/margins": 2.4808454513549805,
+      "rewards/rejected": -0.6282044649124146,
+      "step": 588
+    },
+    {
+      "epoch": 37.59,
+      "grad_norm": 2.013282299041748,
+      "learning_rate": 4.4990420043366415e-07,
+      "logits/chosen": -2.6183061599731445,
+      "logits/rejected": -2.6616299152374268,
+      "logps/chosen": -112.98906707763672,
+      "logps/rejected": -105.28535461425781,
+      "loss": 0.1353,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7617923021316528,
+      "rewards/margins": 2.430490016937256,
+      "rewards/rejected": -0.6686975359916687,
+      "step": 592
+    },
+    {
+      "epoch": 37.84,
+      "grad_norm": 2.2207536697387695,
+      "learning_rate": 4.492235418527227e-07,
+      "logits/chosen": -2.6781811714172363,
+      "logits/rejected": -2.6666665077209473,
+      "logps/chosen": -116.05792999267578,
+      "logps/rejected": -107.16488647460938,
+      "loss": 0.1468,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7365107536315918,
+      "rewards/margins": 2.3593077659606934,
+      "rewards/rejected": -0.6227970719337463,
+      "step": 596
+    },
+    {
+      "epoch": 38.1,
+      "grad_norm": 2.3460826873779297,
+      "learning_rate": 4.48538812562817e-07,
+      "logits/chosen": -2.63061785697937,
+      "logits/rejected": -2.6308352947235107,
+      "logps/chosen": -111.25851440429688,
+      "logps/rejected": -103.23674011230469,
+      "loss": 0.1414,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.6231989860534668,
+      "rewards/margins": 2.478404998779297,
+      "rewards/rejected": -0.8552060127258301,
+      "step": 600
+    },
+    {
+      "epoch": 38.1,
+      "eval_logits/chosen": -2.7357699871063232,
+      "eval_logits/rejected": -2.804048538208008,
+      "eval_logps/chosen": -121.1922607421875,
+      "eval_logps/rejected": -100.1863021850586,
+      "eval_loss": 0.38548800349235535,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.2877070903778076,
+      "eval_rewards/margins": 1.495449423789978,
+      "eval_rewards/rejected": -0.207742378115654,
+      "eval_runtime": 19.3741,
+      "eval_samples_per_second": 1.445,
+      "eval_steps_per_second": 1.445,
+      "step": 600
+    },
+    {
+      "epoch": 38.35,
+      "grad_norm": 2.2148759365081787,
+      "learning_rate": 4.4785002655493244e-07,
+      "logits/chosen": -2.68538236618042,
+      "logits/rejected": -2.721736431121826,
+      "logps/chosen": -115.57979583740234,
+      "logps/rejected": -107.06050872802734,
+      "loss": 0.1434,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.6038768291473389,
+      "rewards/margins": 2.4257020950317383,
+      "rewards/rejected": -0.8218252658843994,
+      "step": 604
+    },
+    {
+      "epoch": 38.6,
+      "grad_norm": 2.045954465866089,
+      "learning_rate": 4.4715719790294473e-07,
+      "logits/chosen": -2.665043592453003,
+      "logits/rejected": -2.6772406101226807,
+      "logps/chosen": -118.3508071899414,
+      "logps/rejected": -105.97952270507812,
+      "loss": 0.1027,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.952754259109497,
+      "rewards/margins": 2.809091329574585,
+      "rewards/rejected": -0.8563370108604431,
+      "step": 608
+    },
+    {
+      "epoch": 38.86,
+      "grad_norm": 3.2380034923553467,
+      "learning_rate": 4.4646034076333254e-07,
+      "logits/chosen": -2.6413214206695557,
+      "logits/rejected": -2.614654064178467,
+      "logps/chosen": -112.02613067626953,
+      "logps/rejected": -107.12358856201172,
+      "loss": 0.1423,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.760948657989502,
+      "rewards/margins": 2.4380459785461426,
+      "rewards/rejected": -0.6770972609519958,
+      "step": 612
+    },
+    {
+      "epoch": 39.11,
+      "grad_norm": 1.8163915872573853,
+      "learning_rate": 4.4575946937488807e-07,
+      "logits/chosen": -2.653783082962036,
+      "logits/rejected": -2.697185516357422,
+      "logps/chosen": -109.09898376464844,
+      "logps/rejected": -105.73826599121094,
+      "loss": 0.1531,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.7506372928619385,
+      "rewards/margins": 2.376413345336914,
+      "rewards/rejected": -0.6257758140563965,
+      "step": 616
+    },
+    {
+      "epoch": 39.37,
+      "grad_norm": 2.3035383224487305,
+      "learning_rate": 4.4505459805842593e-07,
+      "logits/chosen": -2.6597530841827393,
+      "logits/rejected": -2.691429615020752,
+      "logps/chosen": -108.01673889160156,
+      "logps/rejected": -106.93608093261719,
+      "loss": 0.1441,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7096694707870483,
+      "rewards/margins": 2.528447389602661,
+      "rewards/rejected": -0.8187777996063232,
+      "step": 620
+    },
+    {
+      "epoch": 39.62,
+      "grad_norm": 2.202437162399292,
+      "learning_rate": 4.44345741216491e-07,
+      "logits/chosen": -2.6261043548583984,
+      "logits/rejected": -2.6052300930023193,
+      "logps/chosen": -114.36175537109375,
+      "logps/rejected": -107.27129364013672,
+      "loss": 0.1432,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.74629807472229,
+      "rewards/margins": 2.387127637863159,
+      "rewards/rejected": -0.6408295035362244,
+      "step": 624
+    },
+    {
+      "epoch": 39.87,
+      "grad_norm": 2.515286684036255,
+      "learning_rate": 4.43632913333064e-07,
+      "logits/chosen": -2.6591076850891113,
+      "logits/rejected": -2.661803722381592,
+      "logps/chosen": -115.1274185180664,
+      "logps/rejected": -105.72164154052734,
+      "loss": 0.1232,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.884786605834961,
+      "rewards/margins": 2.571641206741333,
+      "rewards/rejected": -0.6868546009063721,
+      "step": 628
+    },
+    {
+      "epoch": 40.13,
+      "grad_norm": 2.1287059783935547,
+      "learning_rate": 4.429161289732649e-07,
+      "logits/chosen": -2.6356821060180664,
+      "logits/rejected": -2.644843101501465,
+      "logps/chosen": -117.84187316894531,
+      "logps/rejected": -108.8236083984375,
+      "loss": 0.1079,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9427728652954102,
+      "rewards/margins": 2.9027700424194336,
+      "rewards/rejected": -0.9599968194961548,
+      "step": 632
+    },
+    {
+      "epoch": 40.38,
+      "grad_norm": 2.836397886276245,
+      "learning_rate": 4.4219540278305647e-07,
+      "logits/chosen": -2.681767463684082,
+      "logits/rejected": -2.682898998260498,
+      "logps/chosen": -108.63336181640625,
+      "logps/rejected": -104.24710845947266,
+      "loss": 0.1326,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.57785165309906,
+      "rewards/margins": 2.463282346725464,
+      "rewards/rejected": -0.8854306936264038,
+      "step": 636
+    },
+    {
+      "epoch": 40.63,
+      "grad_norm": 2.018007516860962,
+      "learning_rate": 4.414707494889439e-07,
+      "logits/chosen": -2.619904041290283,
+      "logits/rejected": -2.6312315464019775,
+      "logps/chosen": -107.43942260742188,
+      "logps/rejected": -104.7081069946289,
+      "loss": 0.1121,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8468282222747803,
+      "rewards/margins": 2.7092161178588867,
+      "rewards/rejected": -0.8623881340026855,
+      "step": 640
+    },
+    {
+      "epoch": 40.63,
+      "eval_logits/chosen": -2.7261669635772705,
+      "eval_logits/rejected": -2.792802572250366,
+      "eval_logps/chosen": -121.07279205322266,
+      "eval_logps/rejected": -100.8832778930664,
+      "eval_loss": 0.3840921223163605,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.2996548414230347,
+      "eval_rewards/margins": 1.5770933628082275,
+      "eval_rewards/rejected": -0.2774384915828705,
+      "eval_runtime": 19.3804,
+      "eval_samples_per_second": 1.445,
+      "eval_steps_per_second": 1.445,
+      "step": 640
+    },
+    {
+      "epoch": 40.89,
+      "grad_norm": 2.168912410736084,
+      "learning_rate": 4.4074218389767466e-07,
+      "logits/chosen": -2.6166744232177734,
+      "logits/rejected": -2.6106488704681396,
+      "logps/chosen": -124.27295684814453,
+      "logps/rejected": -111.69468688964844,
+      "loss": 0.1185,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9064750671386719,
+      "rewards/margins": 2.79134464263916,
+      "rewards/rejected": -0.8848695755004883,
+      "step": 644
+    },
+    {
+      "epoch": 41.14,
+      "grad_norm": 1.7980531454086304,
+      "learning_rate": 4.400097208959356e-07,
+      "logits/chosen": -2.6521995067596436,
+      "logits/rejected": -2.704921245574951,
+      "logps/chosen": -114.37103271484375,
+      "logps/rejected": -113.26276397705078,
+      "loss": 0.1368,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.8770157098770142,
+      "rewards/margins": 2.7274460792541504,
+      "rewards/rejected": -0.8504302501678467,
+      "step": 648
+    },
+    {
+      "epoch": 41.4,
+      "grad_norm": 2.447611093521118,
+      "learning_rate": 4.3927337545004894e-07,
+      "logits/chosen": -2.6812400817871094,
+      "logits/rejected": -2.725090503692627,
+      "logps/chosen": -112.9039535522461,
+      "logps/rejected": -103.30266571044922,
+      "loss": 0.119,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7921515703201294,
+      "rewards/margins": 2.6354293823242188,
+      "rewards/rejected": -0.8432780504226685,
+      "step": 652
+    },
+    {
+      "epoch": 41.65,
+      "grad_norm": 2.0098862648010254,
+      "learning_rate": 4.385331626056663e-07,
+      "logits/chosen": -2.6464550495147705,
+      "logits/rejected": -2.60159969329834,
+      "logps/chosen": -118.65347290039062,
+      "logps/rejected": -106.13660430908203,
+      "loss": 0.1482,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.773016333580017,
+      "rewards/margins": 2.4340577125549316,
+      "rewards/rejected": -0.661041259765625,
+      "step": 656
+    },
+    {
+      "epoch": 41.9,
+      "grad_norm": 2.093533515930176,
+      "learning_rate": 4.377890974874613e-07,
+      "logits/chosen": -2.652855157852173,
+      "logits/rejected": -2.6764514446258545,
+      "logps/chosen": -110.1133804321289,
+      "logps/rejected": -112.76753997802734,
+      "loss": 0.1062,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.795628309249878,
+      "rewards/margins": 2.800370693206787,
+      "rewards/rejected": -1.0047426223754883,
+      "step": 660
+    },
+    {
+      "epoch": 42.16,
+      "grad_norm": 1.8786094188690186,
+      "learning_rate": 4.370411952988206e-07,
+      "logits/chosen": -2.5962324142456055,
+      "logits/rejected": -2.580698251724243,
+      "logps/chosen": -116.74523162841797,
+      "logps/rejected": -108.6583251953125,
+      "loss": 0.1241,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7898542881011963,
+      "rewards/margins": 2.6399893760681152,
+      "rewards/rejected": -0.8501349687576294,
+      "step": 664
+    },
+    {
+      "epoch": 42.41,
+      "grad_norm": 1.1881071329116821,
+      "learning_rate": 4.362894713215334e-07,
+      "logits/chosen": -2.642057418823242,
+      "logits/rejected": -2.6707074642181396,
+      "logps/chosen": -114.59297180175781,
+      "logps/rejected": -107.72032165527344,
+      "loss": 0.1115,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9477055072784424,
+      "rewards/margins": 2.8435428142547607,
+      "rewards/rejected": -0.8958378434181213,
+      "step": 668
+    },
+    {
+      "epoch": 42.67,
+      "grad_norm": 2.0914270877838135,
+      "learning_rate": 4.355339409154788e-07,
+      "logits/chosen": -2.650442123413086,
+      "logits/rejected": -2.647216558456421,
+      "logps/chosen": -108.08013916015625,
+      "logps/rejected": -103.83348083496094,
+      "loss": 0.093,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8842676877975464,
+      "rewards/margins": 2.963129997253418,
+      "rewards/rejected": -1.0788623094558716,
+      "step": 672
+    },
+    {
+      "epoch": 42.92,
+      "grad_norm": 2.257300615310669,
+      "learning_rate": 4.347746195183122e-07,
+      "logits/chosen": -2.6150929927825928,
+      "logits/rejected": -2.6115312576293945,
+      "logps/chosen": -113.52247619628906,
+      "logps/rejected": -110.75169372558594,
+      "loss": 0.1205,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.8281488418579102,
+      "rewards/margins": 2.7573764324188232,
+      "rewards/rejected": -0.9292274117469788,
+      "step": 676
+    },
+    {
+      "epoch": 43.17,
+      "grad_norm": 1.82343327999115,
+      "learning_rate": 4.340115226451501e-07,
+      "logits/chosen": -2.6201157569885254,
+      "logits/rejected": -2.6617937088012695,
+      "logps/chosen": -108.35755157470703,
+      "logps/rejected": -107.59652709960938,
+      "loss": 0.0976,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8024065494537354,
+      "rewards/margins": 2.9102468490600586,
+      "rewards/rejected": -1.1078401803970337,
+      "step": 680
+    },
+    {
+      "epoch": 43.17,
+      "eval_logits/chosen": -2.7156684398651123,
+      "eval_logits/rejected": -2.7805325984954834,
+      "eval_logps/chosen": -121.03025817871094,
+      "eval_logps/rejected": -101.69651794433594,
+      "eval_loss": 0.38208508491516113,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.3039093017578125,
+      "eval_rewards/margins": 1.6626710891723633,
+      "eval_rewards/rejected": -0.358761727809906,
+      "eval_runtime": 19.3666,
+      "eval_samples_per_second": 1.446,
+      "eval_steps_per_second": 1.446,
+      "step": 680
+    },
+    {
+      "epoch": 43.43,
+      "grad_norm": 1.9972604513168335,
+      "learning_rate": 4.3324466588825223e-07,
+      "logits/chosen": -2.6469597816467285,
+      "logits/rejected": -2.6479005813598633,
+      "logps/chosen": -117.90531921386719,
+      "logps/rejected": -106.33170318603516,
+      "loss": 0.1079,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8468865156173706,
+      "rewards/margins": 2.780641555786133,
+      "rewards/rejected": -0.9337549805641174,
+      "step": 684
+    },
+    {
+      "epoch": 43.68,
+      "grad_norm": 1.7684298753738403,
+      "learning_rate": 4.3247406491670435e-07,
+      "logits/chosen": -2.653805732727051,
+      "logits/rejected": -2.658804178237915,
+      "logps/chosen": -108.92557525634766,
+      "logps/rejected": -110.38461303710938,
+      "loss": 0.1272,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9131863117218018,
+      "rewards/margins": 2.63901948928833,
+      "rewards/rejected": -0.7258330583572388,
+      "step": 688
+    },
+    {
+      "epoch": 43.94,
+      "grad_norm": 1.980388879776001,
+      "learning_rate": 4.3169973547609644e-07,
+      "logits/chosen": -2.637269973754883,
+      "logits/rejected": -2.6658825874328613,
+      "logps/chosen": -110.72245788574219,
+      "logps/rejected": -109.98155975341797,
+      "loss": 0.1142,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.804459810256958,
+      "rewards/margins": 2.8206331729888916,
+      "rewards/rejected": -1.0161734819412231,
+      "step": 692
+    },
+    {
+      "epoch": 44.19,
+      "grad_norm": 1.6069421768188477,
+      "learning_rate": 4.3092169338820245e-07,
+      "logits/chosen": -2.6507728099823,
+      "logits/rejected": -2.5921573638916016,
+      "logps/chosen": -115.70768737792969,
+      "logps/rejected": -117.3571548461914,
+      "loss": 0.1211,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.852295994758606,
+      "rewards/margins": 2.9615478515625,
+      "rewards/rejected": -1.1092519760131836,
+      "step": 696
+    },
+    {
+      "epoch": 44.44,
+      "grad_norm": 2.384201765060425,
+      "learning_rate": 4.30139954550656e-07,
+      "logits/chosen": -2.6414434909820557,
+      "logits/rejected": -2.638594627380371,
+      "logps/chosen": -112.53083038330078,
+      "logps/rejected": -103.69336700439453,
+      "loss": 0.0863,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9861321449279785,
+      "rewards/margins": 3.037813663482666,
+      "rewards/rejected": -1.0516817569732666,
+      "step": 700
+    },
+    {
+      "epoch": 44.7,
+      "grad_norm": 2.6236279010772705,
+      "learning_rate": 4.293545349366261e-07,
+      "logits/chosen": -2.627302646636963,
+      "logits/rejected": -2.648130416870117,
+      "logps/chosen": -117.52839660644531,
+      "logps/rejected": -113.75141143798828,
+      "loss": 0.1266,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7409615516662598,
+      "rewards/margins": 2.609776258468628,
+      "rewards/rejected": -0.8688147068023682,
+      "step": 704
+    },
+    {
+      "epoch": 44.95,
+      "grad_norm": 1.3566991090774536,
+      "learning_rate": 4.2856545059449054e-07,
+      "logits/chosen": -2.609771251678467,
+      "logits/rejected": -2.649744749069214,
+      "logps/chosen": -110.12776947021484,
+      "logps/rejected": -104.61019897460938,
+      "loss": 0.0946,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9009833335876465,
+      "rewards/margins": 3.0006396770477295,
+      "rewards/rejected": -1.099656343460083,
+      "step": 708
+    },
+    {
+      "epoch": 45.21,
+      "grad_norm": 1.4097861051559448,
+      "learning_rate": 4.27772717647508e-07,
+      "logits/chosen": -2.639524459838867,
+      "logits/rejected": -2.6454386711120605,
+      "logps/chosen": -110.92047119140625,
+      "logps/rejected": -104.04742431640625,
+      "loss": 0.1045,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7894549369812012,
+      "rewards/margins": 2.7535037994384766,
+      "rewards/rejected": -0.9640489816665649,
+      "step": 712
+    },
+    {
+      "epoch": 45.46,
+      "grad_norm": 2.119607448577881,
+      "learning_rate": 4.2697635229348874e-07,
+      "logits/chosen": -2.6511929035186768,
+      "logits/rejected": -2.633582353591919,
+      "logps/chosen": -109.69573974609375,
+      "logps/rejected": -110.53610229492188,
+      "loss": 0.1163,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7527060508728027,
+      "rewards/margins": 2.6145551204681396,
+      "rewards/rejected": -0.8618494868278503,
+      "step": 716
+    },
+    {
+      "epoch": 45.71,
+      "grad_norm": 1.88676917552948,
+      "learning_rate": 4.261763708044632e-07,
+      "logits/chosen": -2.637509822845459,
+      "logits/rejected": -2.6555514335632324,
+      "logps/chosen": -114.49153137207031,
+      "logps/rejected": -110.72467041015625,
+      "loss": 0.1129,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.9959712028503418,
+      "rewards/margins": 3.11696720123291,
+      "rewards/rejected": -1.1209962368011475,
+      "step": 720
+    },
+    {
+      "epoch": 45.71,
+      "eval_logits/chosen": -2.704455614089966,
+      "eval_logits/rejected": -2.7675421237945557,
+      "eval_logps/chosen": -121.06656646728516,
+      "eval_logps/rejected": -102.52320098876953,
+      "eval_loss": 0.38231486082077026,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.3002774715423584,
+      "eval_rewards/margins": 1.741708517074585,
+      "eval_rewards/rejected": -0.44143104553222656,
+      "eval_runtime": 19.3804,
+      "eval_samples_per_second": 1.445,
+      "eval_steps_per_second": 1.445,
+      "step": 720
+    },
+    {
+      "epoch": 45.97,
+      "grad_norm": 1.0807371139526367,
+      "learning_rate": 4.2537278952635036e-07,
+      "logits/chosen": -2.653682231903076,
+      "logits/rejected": -2.6631548404693604,
+      "logps/chosen": -112.33302307128906,
+      "logps/rejected": -112.11505889892578,
+      "loss": 0.0763,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9477115869522095,
+      "rewards/margins": 3.1752381324768066,
+      "rewards/rejected": -1.227526307106018,
+      "step": 724
+    },
+    {
+      "epoch": 46.22,
+      "grad_norm": 1.5240466594696045,
+      "learning_rate": 4.245656248786228e-07,
+      "logits/chosen": -2.5998778343200684,
+      "logits/rejected": -2.628650188446045,
+      "logps/chosen": -116.5973892211914,
+      "logps/rejected": -119.11909484863281,
+      "loss": 0.0753,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.059324026107788,
+      "rewards/margins": 3.2868502140045166,
+      "rewards/rejected": -1.2275261878967285,
+      "step": 728
+    },
+    {
+      "epoch": 46.48,
+      "grad_norm": 1.6910673379898071,
+      "learning_rate": 4.237548933539718e-07,
+      "logits/chosen": -2.609449863433838,
+      "logits/rejected": -2.6589772701263428,
+      "logps/chosen": -110.6764907836914,
+      "logps/rejected": -105.00558471679688,
+      "loss": 0.1179,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.9273223876953125,
+      "rewards/margins": 2.8337433338165283,
+      "rewards/rejected": -0.9064207077026367,
+      "step": 732
+    },
+    {
+      "epoch": 46.73,
+      "grad_norm": 1.9393585920333862,
+      "learning_rate": 4.229406115179702e-07,
+      "logits/chosen": -2.674577474594116,
+      "logits/rejected": -2.6538712978363037,
+      "logps/chosen": -114.55770874023438,
+      "logps/rejected": -111.56117248535156,
+      "loss": 0.1062,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.6356512308120728,
+      "rewards/margins": 2.7821154594421387,
+      "rewards/rejected": -1.146464467048645,
+      "step": 736
+    },
+    {
+      "epoch": 46.98,
+      "grad_norm": 1.7621327638626099,
+      "learning_rate": 4.221227960087338e-07,
+      "logits/chosen": -2.6068484783172607,
+      "logits/rejected": -2.5801186561584473,
+      "logps/chosen": -109.57792663574219,
+      "logps/rejected": -106.45953369140625,
+      "loss": 0.0958,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8922866582870483,
+      "rewards/margins": 3.114880323410034,
+      "rewards/rejected": -1.2225935459136963,
+      "step": 740
+    },
+    {
+      "epoch": 47.24,
+      "grad_norm": 1.7458181381225586,
+      "learning_rate": 4.213014635365815e-07,
+      "logits/chosen": -2.6573941707611084,
+      "logits/rejected": -2.653240919113159,
+      "logps/chosen": -116.97773742675781,
+      "logps/rejected": -108.12200164794922,
+      "loss": 0.0816,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0080180168151855,
+      "rewards/margins": 3.296840190887451,
+      "rewards/rejected": -1.2888221740722656,
+      "step": 744
+    },
+    {
+      "epoch": 47.49,
+      "grad_norm": 1.9578934907913208,
+      "learning_rate": 4.20476630883694e-07,
+      "logits/chosen": -2.5878279209136963,
+      "logits/rejected": -2.595855712890625,
+      "logps/chosen": -115.75021362304688,
+      "logps/rejected": -112.4002685546875,
+      "loss": 0.075,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9529811143875122,
+      "rewards/margins": 3.2216196060180664,
+      "rewards/rejected": -1.2686389684677124,
+      "step": 748
+    },
+    {
+      "epoch": 47.75,
+      "grad_norm": 1.6000980138778687,
+      "learning_rate": 4.196483149037706e-07,
+      "logits/chosen": -2.632962226867676,
+      "logits/rejected": -2.6356112957000732,
+      "logps/chosen": -110.47245788574219,
+      "logps/rejected": -111.49674224853516,
+      "loss": 0.1139,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.7682106494903564,
+      "rewards/margins": 2.8430519104003906,
+      "rewards/rejected": -1.074841022491455,
+      "step": 752
+    },
+    {
+      "epoch": 48.0,
+      "grad_norm": 1.9791028499603271,
+      "learning_rate": 4.188165325216849e-07,
+      "logits/chosen": -2.6407368183135986,
+      "logits/rejected": -2.6439175605773926,
+      "logps/chosen": -107.70930480957031,
+      "logps/rejected": -111.42974853515625,
+      "loss": 0.1044,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9136321544647217,
+      "rewards/margins": 3.008749485015869,
+      "rewards/rejected": -1.095117211341858,
+      "step": 756
+    },
+    {
+      "epoch": 48.25,
+      "grad_norm": 1.486619234085083,
+      "learning_rate": 4.179813007331393e-07,
+      "logits/chosen": -2.638111114501953,
+      "logits/rejected": -2.6660499572753906,
+      "logps/chosen": -112.31642150878906,
+      "logps/rejected": -114.76564025878906,
+      "loss": 0.1037,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7412384748458862,
+      "rewards/margins": 2.972817897796631,
+      "rewards/rejected": -1.2315791845321655,
+      "step": 760
+    },
+    {
+      "epoch": 48.25,
+      "eval_logits/chosen": -2.69281268119812,
+      "eval_logits/rejected": -2.754284143447876,
+      "eval_logps/chosen": -121.21448516845703,
+      "eval_logps/rejected": -103.48255920410156,
+      "eval_loss": 0.3811449110507965,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.2854851484298706,
+      "eval_rewards/margins": 1.8228529691696167,
+      "eval_rewards/rejected": -0.5373677015304565,
+      "eval_runtime": 19.4147,
+      "eval_samples_per_second": 1.442,
+      "eval_steps_per_second": 1.442,
+      "step": 760
+    },
+    {
+      "epoch": 48.51,
+      "grad_norm": 1.6442840099334717,
+      "learning_rate": 4.171426366043172e-07,
+      "logits/chosen": -2.5812575817108154,
+      "logits/rejected": -2.6027064323425293,
+      "logps/chosen": -109.93325805664062,
+      "logps/rejected": -106.68315887451172,
+      "loss": 0.1014,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 2.025794506072998,
+      "rewards/margins": 2.979997158050537,
+      "rewards/rejected": -0.9542028903961182,
+      "step": 764
+    },
+    {
+      "epoch": 48.76,
+      "grad_norm": 1.7033922672271729,
+      "learning_rate": 4.163005572715348e-07,
+      "logits/chosen": -2.6315484046936035,
+      "logits/rejected": -2.652425765991211,
+      "logps/chosen": -114.54792785644531,
+      "logps/rejected": -113.0596923828125,
+      "loss": 0.0923,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9352294206619263,
+      "rewards/margins": 3.1911542415618896,
+      "rewards/rejected": -1.2559247016906738,
+      "step": 768
+    },
+    {
+      "epoch": 49.02,
+      "grad_norm": 1.4940606355667114,
+      "learning_rate": 4.154550799408906e-07,
+      "logits/chosen": -2.6459126472473145,
+      "logits/rejected": -2.59468936920166,
+      "logps/chosen": -112.78709411621094,
+      "logps/rejected": -111.69085693359375,
+      "loss": 0.0632,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9187610149383545,
+      "rewards/margins": 3.4088921546936035,
+      "rewards/rejected": -1.4901307821273804,
+      "step": 772
+    },
+    {
+      "epoch": 49.27,
+      "grad_norm": 1.6349976062774658,
+      "learning_rate": 4.1460622188791386e-07,
+      "logits/chosen": -2.626100778579712,
+      "logits/rejected": -2.6048431396484375,
+      "logps/chosen": -114.90809631347656,
+      "logps/rejected": -110.02332305908203,
+      "loss": 0.0862,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.9543546438217163,
+      "rewards/margins": 3.2480387687683105,
+      "rewards/rejected": -1.2936842441558838,
+      "step": 776
+    },
+    {
+      "epoch": 49.52,
+      "grad_norm": 1.4356471300125122,
+      "learning_rate": 4.13754000457212e-07,
+      "logits/chosen": -2.6456077098846436,
+      "logits/rejected": -2.676055908203125,
+      "logps/chosen": -103.83885955810547,
+      "logps/rejected": -109.37602233886719,
+      "loss": 0.082,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8857074975967407,
+      "rewards/margins": 3.213439702987671,
+      "rewards/rejected": -1.3277320861816406,
+      "step": 780
+    },
+    {
+      "epoch": 49.78,
+      "grad_norm": 1.7946281433105469,
+      "learning_rate": 4.128984330621156e-07,
+      "logits/chosen": -2.6110281944274902,
+      "logits/rejected": -2.605884075164795,
+      "logps/chosen": -116.66584777832031,
+      "logps/rejected": -114.77125549316406,
+      "loss": 0.0825,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0007410049438477,
+      "rewards/margins": 3.1677379608154297,
+      "rewards/rejected": -1.1669968366622925,
+      "step": 784
+    },
+    {
+      "epoch": 50.03,
+      "grad_norm": 2.0904555320739746,
+      "learning_rate": 4.1203953718432304e-07,
+      "logits/chosen": -2.5824971199035645,
+      "logits/rejected": -2.5949392318725586,
+      "logps/chosen": -113.75666809082031,
+      "logps/rejected": -112.34939575195312,
+      "loss": 0.0968,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8396023511886597,
+      "rewards/margins": 3.167187213897705,
+      "rewards/rejected": -1.3275845050811768,
+      "step": 788
+    },
+    {
+      "epoch": 50.29,
+      "grad_norm": 1.8408178091049194,
+      "learning_rate": 4.1117733037354313e-07,
+      "logits/chosen": -2.5902395248413086,
+      "logits/rejected": -2.590879440307617,
+      "logps/chosen": -116.93998718261719,
+      "logps/rejected": -112.17693328857422,
+      "loss": 0.0719,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.067030429840088,
+      "rewards/margins": 3.2486207485198975,
+      "rewards/rejected": -1.1815906763076782,
+      "step": 792
+    },
+    {
+      "epoch": 50.54,
+      "grad_norm": 1.2143826484680176,
+      "learning_rate": 4.1031183024713657e-07,
+      "logits/chosen": -2.6065564155578613,
+      "logits/rejected": -2.5807137489318848,
+      "logps/chosen": -116.70026397705078,
+      "logps/rejected": -109.03843688964844,
+      "loss": 0.0901,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 2.0011649131774902,
+      "rewards/margins": 3.269892454147339,
+      "rewards/rejected": -1.2687275409698486,
+      "step": 796
+    },
+    {
+      "epoch": 50.79,
+      "grad_norm": 1.2470941543579102,
+      "learning_rate": 4.0944305448975594e-07,
+      "logits/chosen": -2.6263880729675293,
+      "logits/rejected": -2.6351208686828613,
+      "logps/chosen": -109.2755126953125,
+      "logps/rejected": -116.7694091796875,
+      "loss": 0.085,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.79188871383667,
+      "rewards/margins": 3.278684616088867,
+      "rewards/rejected": -1.4867955446243286,
+      "step": 800
+    },
+    {
+      "epoch": 50.79,
+      "eval_logits/chosen": -2.681551694869995,
+      "eval_logits/rejected": -2.741281270980835,
+      "eval_logps/chosen": -121.30486297607422,
+      "eval_logps/rejected": -104.38188171386719,
+      "eval_loss": 0.38058552145957947,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.2764482498168945,
+      "eval_rewards/margins": 1.9037466049194336,
+      "eval_rewards/rejected": -0.6272983551025391,
+      "eval_runtime": 19.3679,
+      "eval_samples_per_second": 1.446,
+      "eval_steps_per_second": 1.446,
+      "step": 800
+    },
+    {
+      "epoch": 51.05,
+      "grad_norm": 2.3190038204193115,
+      "learning_rate": 4.085710208529843e-07,
+      "logits/chosen": -2.6023600101470947,
+      "logits/rejected": -2.6153862476348877,
+      "logps/chosen": -105.47126007080078,
+      "logps/rejected": -109.3538818359375,
+      "loss": 0.0898,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8685368299484253,
+      "rewards/margins": 3.1377530097961426,
+      "rewards/rejected": -1.2692162990570068,
+      "step": 804
+    },
+    {
+      "epoch": 51.3,
+      "grad_norm": 1.2653313875198364,
+      "learning_rate": 4.076957471549728e-07,
+      "logits/chosen": -2.6036365032196045,
+      "logits/rejected": -2.623936414718628,
+      "logps/chosen": -112.60127258300781,
+      "logps/rejected": -108.7287368774414,
+      "loss": 0.0865,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9202613830566406,
+      "rewards/margins": 3.164761543273926,
+      "rewards/rejected": -1.2445003986358643,
+      "step": 808
+    },
+    {
+      "epoch": 51.56,
+      "grad_norm": 1.1972923278808594,
+      "learning_rate": 4.068172512800759e-07,
+      "logits/chosen": -2.5859198570251465,
+      "logits/rejected": -2.5816516876220703,
+      "logps/chosen": -110.08045959472656,
+      "logps/rejected": -117.87619018554688,
+      "loss": 0.0605,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0786402225494385,
+      "rewards/margins": 3.6814520359039307,
+      "rewards/rejected": -1.6028118133544922,
+      "step": 812
+    },
+    {
+      "epoch": 51.81,
+      "grad_norm": 1.5057625770568848,
+      "learning_rate": 4.059355511784868e-07,
+      "logits/chosen": -2.619558572769165,
+      "logits/rejected": -2.6305971145629883,
+      "logps/chosen": -114.13056945800781,
+      "logps/rejected": -113.92069244384766,
+      "loss": 0.0933,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.7876722812652588,
+      "rewards/margins": 3.144243001937866,
+      "rewards/rejected": -1.3565707206726074,
+      "step": 816
+    },
+    {
+      "epoch": 52.06,
+      "grad_norm": 1.5957279205322266,
+      "learning_rate": 4.0505066486587e-07,
+      "logits/chosen": -2.638758659362793,
+      "logits/rejected": -2.615614175796509,
+      "logps/chosen": -117.18038940429688,
+      "logps/rejected": -118.73646545410156,
+      "loss": 0.0666,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.000675678253174,
+      "rewards/margins": 3.431554079055786,
+      "rewards/rejected": -1.4308786392211914,
+      "step": 820
+    },
+    {
+      "epoch": 52.32,
+      "grad_norm": 1.7117719650268555,
+      "learning_rate": 4.041626104229936e-07,
+      "logits/chosen": -2.561399221420288,
+      "logits/rejected": -2.5965375900268555,
+      "logps/chosen": -108.37356567382812,
+      "logps/rejected": -109.40271759033203,
+      "loss": 0.0615,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.173064708709717,
+      "rewards/margins": 3.6536784172058105,
+      "rewards/rejected": -1.4806135892868042,
+      "step": 824
+    },
+    {
+      "epoch": 52.57,
+      "grad_norm": 1.3572646379470825,
+      "learning_rate": 4.032714059953595e-07,
+      "logits/chosen": -2.6196985244750977,
+      "logits/rejected": -2.614413022994995,
+      "logps/chosen": -109.2529067993164,
+      "logps/rejected": -104.56177520751953,
+      "loss": 0.092,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.9012202024459839,
+      "rewards/margins": 3.18809175491333,
+      "rewards/rejected": -1.2868719100952148,
+      "step": 828
+    },
+    {
+      "epoch": 52.83,
+      "grad_norm": 1.784955382347107,
+      "learning_rate": 4.0237706979283305e-07,
+      "logits/chosen": -2.6027987003326416,
+      "logits/rejected": -2.6091623306274414,
+      "logps/chosen": -116.41980743408203,
+      "logps/rejected": -121.5534896850586,
+      "loss": 0.0837,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.864450216293335,
+      "rewards/margins": 3.336150646209717,
+      "rewards/rejected": -1.4717004299163818,
+      "step": 832
+    },
+    {
+      "epoch": 53.08,
+      "grad_norm": 1.603405237197876,
+      "learning_rate": 4.014796200892706e-07,
+      "logits/chosen": -2.59523344039917,
+      "logits/rejected": -2.59432053565979,
+      "logps/chosen": -108.4609603881836,
+      "logps/rejected": -112.68742370605469,
+      "loss": 0.071,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.829506278038025,
+      "rewards/margins": 3.3810737133026123,
+      "rewards/rejected": -1.5515676736831665,
+      "step": 836
+    },
+    {
+      "epoch": 53.33,
+      "grad_norm": 1.8830804824829102,
+      "learning_rate": 4.005790752221464e-07,
+      "logits/chosen": -2.6272573471069336,
+      "logits/rejected": -2.627049446105957,
+      "logps/chosen": -113.10144805908203,
+      "logps/rejected": -108.62581634521484,
+      "loss": 0.0714,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.913501262664795,
+      "rewards/margins": 3.577221632003784,
+      "rewards/rejected": -1.663720726966858,
+      "step": 840
+    },
+    {
+      "epoch": 53.33,
+      "eval_logits/chosen": -2.668611526489258,
+      "eval_logits/rejected": -2.726428508758545,
+      "eval_logps/chosen": -121.56246185302734,
+      "eval_logps/rejected": -105.41184997558594,
+      "eval_loss": 0.381181925535202,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.2506877183914185,
+      "eval_rewards/margins": 1.9809836149215698,
+      "eval_rewards/rejected": -0.7302957773208618,
+      "eval_runtime": 19.382,
+      "eval_samples_per_second": 1.445,
+      "eval_steps_per_second": 1.445,
+      "step": 840
+    },
+    {
+      "epoch": 53.59,
+      "grad_norm": 1.5219979286193848,
+      "learning_rate": 3.996754535921777e-07,
+      "logits/chosen": -2.594785451889038,
+      "logits/rejected": -2.600487470626831,
+      "logps/chosen": -106.41715240478516,
+      "logps/rejected": -120.44597625732422,
+      "loss": 0.0765,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9574780464172363,
+      "rewards/margins": 3.235558032989502,
+      "rewards/rejected": -1.2780797481536865,
+      "step": 844
+    },
+    {
+      "epoch": 53.84,
+      "grad_norm": 0.8998934626579285,
+      "learning_rate": 3.987687736629487e-07,
+      "logits/chosen": -2.5739502906799316,
+      "logits/rejected": -2.540557861328125,
+      "logps/chosen": -116.07780456542969,
+      "logps/rejected": -113.49636840820312,
+      "loss": 0.0588,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.980076551437378,
+      "rewards/margins": 3.6954421997070312,
+      "rewards/rejected": -1.7153657674789429,
+      "step": 848
+    },
+    {
+      "epoch": 54.1,
+      "grad_norm": 1.3017789125442505,
+      "learning_rate": 3.9785905396053377e-07,
+      "logits/chosen": -2.5990142822265625,
+      "logits/rejected": -2.6168363094329834,
+      "logps/chosen": -111.93960571289062,
+      "logps/rejected": -116.76874542236328,
+      "loss": 0.0872,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 2.0168068408966064,
+      "rewards/margins": 3.3573083877563477,
+      "rewards/rejected": -1.340501308441162,
+      "step": 852
+    },
+    {
+      "epoch": 54.35,
+      "grad_norm": 1.2396210432052612,
+      "learning_rate": 3.9694631307311825e-07,
+      "logits/chosen": -2.5667662620544434,
+      "logits/rejected": -2.5850954055786133,
+      "logps/chosen": -111.22042846679688,
+      "logps/rejected": -111.89250183105469,
+      "loss": 0.0731,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9918500185012817,
+      "rewards/margins": 3.535062313079834,
+      "rewards/rejected": -1.5432121753692627,
+      "step": 856
+    },
+    {
+      "epoch": 54.6,
+      "grad_norm": 1.335559606552124,
+      "learning_rate": 3.960305696506192e-07,
+      "logits/chosen": -2.5854077339172363,
+      "logits/rejected": -2.5688228607177734,
+      "logps/chosen": -116.70706939697266,
+      "logps/rejected": -110.84555053710938,
+      "loss": 0.0528,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.941176414489746,
+      "rewards/margins": 3.603893518447876,
+      "rewards/rejected": -1.6627171039581299,
+      "step": 860
+    },
+    {
+      "epoch": 54.86,
+      "grad_norm": 1.6855989694595337,
+      "learning_rate": 3.95111842404304e-07,
+      "logits/chosen": -2.5832574367523193,
+      "logits/rejected": -2.5922787189483643,
+      "logps/chosen": -115.02678680419922,
+      "logps/rejected": -123.42115783691406,
+      "loss": 0.0682,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9162797927856445,
+      "rewards/margins": 3.6613926887512207,
+      "rewards/rejected": -1.7451128959655762,
+      "step": 864
+    },
+    {
+      "epoch": 55.11,
+      "grad_norm": 1.599083423614502,
+      "learning_rate": 3.94190150106408e-07,
+      "logits/chosen": -2.6082019805908203,
+      "logits/rejected": -2.583937168121338,
+      "logps/chosen": -106.68289947509766,
+      "logps/rejected": -116.42233276367188,
+      "loss": 0.0741,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8997607231140137,
+      "rewards/margins": 3.2817296981811523,
+      "rewards/rejected": -1.3819692134857178,
+      "step": 868
+    },
+    {
+      "epoch": 55.37,
+      "grad_norm": 1.4026548862457275,
+      "learning_rate": 3.9326551158975124e-07,
+      "logits/chosen": -2.575761079788208,
+      "logits/rejected": -2.6161446571350098,
+      "logps/chosen": -109.90505981445312,
+      "logps/rejected": -113.70225524902344,
+      "loss": 0.0556,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.069807291030884,
+      "rewards/margins": 3.6844887733459473,
+      "rewards/rejected": -1.6146814823150635,
+      "step": 872
+    },
+    {
+      "epoch": 55.62,
+      "grad_norm": 1.6232682466506958,
+      "learning_rate": 3.923379457473534e-07,
+      "logits/chosen": -2.569347858428955,
+      "logits/rejected": -2.5830793380737305,
+      "logps/chosen": -115.759033203125,
+      "logps/rejected": -114.3780288696289,
+      "loss": 0.0651,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.960666298866272,
+      "rewards/margins": 3.5167884826660156,
+      "rewards/rejected": -1.5561223030090332,
+      "step": 876
+    },
+    {
+      "epoch": 55.87,
+      "grad_norm": 1.7684590816497803,
+      "learning_rate": 3.914074715320479e-07,
+      "logits/chosen": -2.5655999183654785,
+      "logits/rejected": -2.5638012886047363,
+      "logps/chosen": -109.46947479248047,
+      "logps/rejected": -118.88379669189453,
+      "loss": 0.0605,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8757377862930298,
+      "rewards/margins": 3.77754807472229,
+      "rewards/rejected": -1.9018104076385498,
+      "step": 880
+    },
+    {
+      "epoch": 55.87,
+      "eval_logits/chosen": -2.656400203704834,
+      "eval_logits/rejected": -2.712291955947876,
+      "eval_logps/chosen": -121.80836486816406,
+      "eval_logps/rejected": -106.49579620361328,
+      "eval_loss": 0.380399227142334,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 1.2260981798171997,
+      "eval_rewards/margins": 2.0647895336151123,
+      "eval_rewards/rejected": -0.8386915922164917,
+      "eval_runtime": 19.382,
+      "eval_samples_per_second": 1.445,
+      "eval_steps_per_second": 1.445,
+      "step": 880
+    },
+    {
+      "epoch": 56.13,
+      "grad_norm": 1.283678412437439,
+      "learning_rate": 3.9047410795609437e-07,
+      "logits/chosen": -2.621877908706665,
+      "logits/rejected": -2.5620386600494385,
+      "logps/chosen": -112.5902328491211,
+      "logps/rejected": -115.07412719726562,
+      "loss": 0.0779,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.8606853485107422,
+      "rewards/margins": 3.527510166168213,
+      "rewards/rejected": -1.6668250560760498,
+      "step": 884
+    },
+    {
+      "epoch": 56.38,
+      "grad_norm": 1.6468217372894287,
+      "learning_rate": 3.8953787409079076e-07,
+      "logits/chosen": -2.571714401245117,
+      "logits/rejected": -2.581312417984009,
+      "logps/chosen": -111.61944580078125,
+      "logps/rejected": -115.42122650146484,
+      "loss": 0.0643,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9630603790283203,
+      "rewards/margins": 3.626448154449463,
+      "rewards/rejected": -1.6633877754211426,
+      "step": 888
+    },
+    {
+      "epoch": 56.63,
+      "grad_norm": 1.1661204099655151,
+      "learning_rate": 3.885987890660827e-07,
+      "logits/chosen": -2.5704822540283203,
+      "logits/rejected": -2.5689477920532227,
+      "logps/chosen": -109.46009063720703,
+      "logps/rejected": -111.1002426147461,
+      "loss": 0.0596,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0093395709991455,
+      "rewards/margins": 3.5506997108459473,
+      "rewards/rejected": -1.5413601398468018,
+      "step": 892
+    },
+    {
+      "epoch": 56.89,
+      "grad_norm": 1.340222716331482,
+      "learning_rate": 3.876568720701737e-07,
+      "logits/chosen": -2.621769428253174,
+      "logits/rejected": -2.6131060123443604,
+      "logps/chosen": -114.04650115966797,
+      "logps/rejected": -119.44813537597656,
+      "loss": 0.0678,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.858720302581787,
+      "rewards/margins": 3.575531244277954,
+      "rewards/rejected": -1.716811180114746,
+      "step": 896
+    },
+    {
+      "epoch": 57.14,
+      "grad_norm": 1.60097074508667,
+      "learning_rate": 3.867121423491324e-07,
+      "logits/chosen": -2.5544419288635254,
+      "logits/rejected": -2.5702197551727295,
+      "logps/chosen": -110.59100341796875,
+      "logps/rejected": -116.37786102294922,
+      "loss": 0.0639,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0666933059692383,
+      "rewards/margins": 3.7600369453430176,
+      "rewards/rejected": -1.6933435201644897,
+      "step": 900
+    },
+    {
+      "epoch": 57.4,
+      "grad_norm": 1.5483367443084717,
+      "learning_rate": 3.857646192064995e-07,
+      "logits/chosen": -2.5642123222351074,
+      "logits/rejected": -2.5501718521118164,
+      "logps/chosen": -110.6575698852539,
+      "logps/rejected": -119.62319946289062,
+      "loss": 0.0632,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.882343053817749,
+      "rewards/margins": 3.3227930068969727,
+      "rewards/rejected": -1.4404499530792236,
+      "step": 904
+    },
+    {
+      "epoch": 57.65,
+      "grad_norm": 1.3593943119049072,
+      "learning_rate": 3.848143220028931e-07,
+      "logits/chosen": -2.6376965045928955,
+      "logits/rejected": -2.618466377258301,
+      "logps/chosen": -115.74939727783203,
+      "logps/rejected": -112.37615966796875,
+      "loss": 0.0644,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9629660844802856,
+      "rewards/margins": 3.574894428253174,
+      "rewards/rejected": -1.6119282245635986,
+      "step": 908
+    },
+    {
+      "epoch": 57.9,
+      "grad_norm": 0.9320303201675415,
+      "learning_rate": 3.8386127015561377e-07,
+      "logits/chosen": -2.5762109756469727,
+      "logits/rejected": -2.5863161087036133,
+      "logps/chosen": -113.18948364257812,
+      "logps/rejected": -116.29486846923828,
+      "loss": 0.0671,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 2.0451762676239014,
+      "rewards/margins": 3.999039649963379,
+      "rewards/rejected": -1.9538631439208984,
+      "step": 912
+    },
+    {
+      "epoch": 58.16,
+      "grad_norm": 1.352599859237671,
+      "learning_rate": 3.82905483138247e-07,
+      "logits/chosen": -2.550541400909424,
+      "logits/rejected": -2.533012866973877,
+      "logps/chosen": -114.77538299560547,
+      "logps/rejected": -119.81204223632812,
+      "loss": 0.0522,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9466917514801025,
+      "rewards/margins": 4.070167541503906,
+      "rewards/rejected": -2.123476028442383,
+      "step": 916
+    },
+    {
+      "epoch": 58.41,
+      "grad_norm": 1.207864761352539,
+      "learning_rate": 3.8194698048026583e-07,
+      "logits/chosen": -2.5619962215423584,
+      "logits/rejected": -2.569873094558716,
+      "logps/chosen": -118.76224517822266,
+      "logps/rejected": -118.89384460449219,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.794409155845642,
+      "rewards/margins": 3.9158012866973877,
+      "rewards/rejected": -2.121392250061035,
+      "step": 920
+    },
+    {
+      "epoch": 58.41,
+      "eval_logits/chosen": -2.6444404125213623,
+      "eval_logits/rejected": -2.6984755992889404,
+      "eval_logps/chosen": -122.1351318359375,
+      "eval_logps/rejected": -107.59381103515625,
+      "eval_loss": 0.38029375672340393,
+      "eval_rewards/accuracies": 0.8214285969734192,
+      "eval_rewards/chosen": 1.193420171737671,
+      "eval_rewards/margins": 2.141911029815674,
+      "eval_rewards/rejected": -0.9484910368919373,
+      "eval_runtime": 19.4042,
+      "eval_samples_per_second": 1.443,
+      "eval_steps_per_second": 1.443,
+      "step": 920
+    },
+    {
+      "epoch": 58.67,
+      "grad_norm": 1.158010482788086,
+      "learning_rate": 3.8098578176663154e-07,
+      "logits/chosen": -2.585352659225464,
+      "logits/rejected": -2.5922374725341797,
+      "logps/chosen": -104.6133804321289,
+      "logps/rejected": -110.61421203613281,
+      "loss": 0.0737,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9587405920028687,
+      "rewards/margins": 3.5576822757720947,
+      "rewards/rejected": -1.598941683769226,
+      "step": 924
+    },
+    {
+      "epoch": 58.92,
+      "grad_norm": 1.1459003686904907,
+      "learning_rate": 3.800219066373936e-07,
+      "logits/chosen": -2.636521816253662,
+      "logits/rejected": -2.6258299350738525,
+      "logps/chosen": -107.60220336914062,
+      "logps/rejected": -120.26052856445312,
+      "loss": 0.0492,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.1595146656036377,
+      "rewards/margins": 3.6657819747924805,
+      "rewards/rejected": -1.5062673091888428,
+      "step": 928
+    },
+    {
+      "epoch": 59.17,
+      "grad_norm": 1.0757678747177124,
+      "learning_rate": 3.7905537478728844e-07,
+      "logits/chosen": -2.5207972526550293,
+      "logits/rejected": -2.497737169265747,
+      "logps/chosen": -112.18553161621094,
+      "logps/rejected": -108.89622497558594,
+      "loss": 0.0525,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0456583499908447,
+      "rewards/margins": 3.8131721019744873,
+      "rewards/rejected": -1.767513632774353,
+      "step": 932
+    },
+    {
+      "epoch": 59.43,
+      "grad_norm": 1.0222171545028687,
+      "learning_rate": 3.780862059653367e-07,
+      "logits/chosen": -2.6137924194335938,
+      "logits/rejected": -2.612893581390381,
+      "logps/chosen": -110.01217651367188,
+      "logps/rejected": -119.69099426269531,
+      "loss": 0.0611,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.8505878448486328,
+      "rewards/margins": 3.9199700355529785,
+      "rewards/rejected": -2.0693817138671875,
+      "step": 936
+    },
+    {
+      "epoch": 59.68,
+      "grad_norm": 1.1177177429199219,
+      "learning_rate": 3.7711441997444016e-07,
+      "logits/chosen": -2.538954257965088,
+      "logits/rejected": -2.535083770751953,
+      "logps/chosen": -117.10211944580078,
+      "logps/rejected": -117.08683776855469,
+      "loss": 0.0614,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.1291184425354004,
+      "rewards/margins": 3.7359395027160645,
+      "rewards/rejected": -1.606820821762085,
+      "step": 940
+    },
+    {
+      "epoch": 59.94,
+      "grad_norm": 1.25459885597229,
+      "learning_rate": 3.7614003667097673e-07,
+      "logits/chosen": -2.5723907947540283,
+      "logits/rejected": -2.600084066390991,
+      "logps/chosen": -106.81842041015625,
+      "logps/rejected": -121.9610824584961,
+      "loss": 0.0534,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7943745851516724,
+      "rewards/margins": 3.866551399230957,
+      "rewards/rejected": -2.0721771717071533,
+      "step": 944
+    },
+    {
+      "epoch": 60.19,
+      "grad_norm": 1.1901063919067383,
+      "learning_rate": 3.7516307596439494e-07,
+      "logits/chosen": -2.5640013217926025,
+      "logits/rejected": -2.5475199222564697,
+      "logps/chosen": -116.98225402832031,
+      "logps/rejected": -117.1365966796875,
+      "loss": 0.0464,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.131293535232544,
+      "rewards/margins": 4.100174903869629,
+      "rewards/rejected": -1.9688811302185059,
+      "step": 948
+    },
+    {
+      "epoch": 60.44,
+      "grad_norm": 1.3003920316696167,
+      "learning_rate": 3.7418355781680707e-07,
+      "logits/chosen": -2.5525896549224854,
+      "logits/rejected": -2.5547983646392822,
+      "logps/chosen": -111.23165893554688,
+      "logps/rejected": -121.36713409423828,
+      "loss": 0.065,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0756218433380127,
+      "rewards/margins": 3.645637035369873,
+      "rewards/rejected": -1.5700148344039917,
+      "step": 952
+    },
+    {
+      "epoch": 60.7,
+      "grad_norm": 1.421809434890747,
+      "learning_rate": 3.7320150224258116e-07,
+      "logits/chosen": -2.5778801441192627,
+      "logits/rejected": -2.585582733154297,
+      "logps/chosen": -110.18714904785156,
+      "logps/rejected": -124.43739318847656,
+      "loss": 0.0465,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7850464582443237,
+      "rewards/margins": 4.019922733306885,
+      "rewards/rejected": -2.2348759174346924,
+      "step": 956
+    },
+    {
+      "epoch": 60.95,
+      "grad_norm": 1.3294024467468262,
+      "learning_rate": 3.7221692930793233e-07,
+      "logits/chosen": -2.5990958213806152,
+      "logits/rejected": -2.5878379344940186,
+      "logps/chosen": -105.92156982421875,
+      "logps/rejected": -110.94420623779297,
+      "loss": 0.0601,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.8963432312011719,
+      "rewards/margins": 3.7428231239318848,
+      "rewards/rejected": -1.846480131149292,
+      "step": 960
+    },
+    {
+      "epoch": 60.95,
+      "eval_logits/chosen": -2.6317238807678223,
+      "eval_logits/rejected": -2.6838760375976562,
+      "eval_logps/chosen": -122.47028350830078,
+      "eval_logps/rejected": -108.6797103881836,
+      "eval_loss": 0.38127702474594116,
+      "eval_rewards/accuracies": 0.8214285969734192,
+      "eval_rewards/chosen": 1.1599055528640747,
+      "eval_rewards/margins": 2.2169876098632812,
+      "eval_rewards/rejected": -1.057082176208496,
+      "eval_runtime": 19.3651,
+      "eval_samples_per_second": 1.446,
+      "eval_steps_per_second": 1.446,
+      "step": 960
+    },
+    {
+      "epoch": 61.21,
+      "grad_norm": 1.2264454364776611,
+      "learning_rate": 3.7122985913051236e-07,
+      "logits/chosen": -2.5557727813720703,
+      "logits/rejected": -2.5267515182495117,
+      "logps/chosen": -121.1830062866211,
+      "logps/rejected": -116.08586120605469,
+      "loss": 0.0411,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0561954975128174,
+      "rewards/margins": 4.135585784912109,
+      "rewards/rejected": -2.079390287399292,
+      "step": 964
+    },
+    {
+      "epoch": 61.46,
+      "grad_norm": 1.0248548984527588,
+      "learning_rate": 3.702403118789992e-07,
+      "logits/chosen": -2.5450968742370605,
+      "logits/rejected": -2.5582926273345947,
+      "logps/chosen": -109.03056335449219,
+      "logps/rejected": -122.5210952758789,
+      "loss": 0.045,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9161659479141235,
+      "rewards/margins": 3.8867244720458984,
+      "rewards/rejected": -1.970558524131775,
+      "step": 968
+    },
+    {
+      "epoch": 61.71,
+      "grad_norm": 1.595377802848816,
+      "learning_rate": 3.692483077726842e-07,
+      "logits/chosen": -2.5482635498046875,
+      "logits/rejected": -2.537240505218506,
+      "logps/chosen": -114.62641906738281,
+      "logps/rejected": -122.08514404296875,
+      "loss": 0.0641,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9590504169464111,
+      "rewards/margins": 3.6032297611236572,
+      "rewards/rejected": -1.644179344177246,
+      "step": 972
+    },
+    {
+      "epoch": 61.97,
+      "grad_norm": 0.8679869771003723,
+      "learning_rate": 3.682538670810596e-07,
+      "logits/chosen": -2.5945587158203125,
+      "logits/rejected": -2.601808786392212,
+      "logps/chosen": -106.33365631103516,
+      "logps/rejected": -115.79863739013672,
+      "loss": 0.0416,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.9806954860687256,
+      "rewards/margins": 4.1988372802734375,
+      "rewards/rejected": -2.218141794204712,
+      "step": 976
+    },
+    {
+      "epoch": 62.22,
+      "grad_norm": 0.6783718466758728,
+      "learning_rate": 3.6725701012340383e-07,
+      "logits/chosen": -2.5552518367767334,
+      "logits/rejected": -2.5349369049072266,
+      "logps/chosen": -115.44186401367188,
+      "logps/rejected": -124.41796112060547,
+      "loss": 0.0501,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0889174938201904,
+      "rewards/margins": 4.224672317504883,
+      "rewards/rejected": -2.1357548236846924,
+      "step": 980
+    },
+    {
+      "epoch": 62.48,
+      "grad_norm": 1.1256548166275024,
+      "learning_rate": 3.6625775726836673e-07,
+      "logits/chosen": -2.5621018409729004,
+      "logits/rejected": -2.541167736053467,
+      "logps/chosen": -112.84183502197266,
+      "logps/rejected": -120.05206298828125,
+      "loss": 0.0485,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.1025571823120117,
+      "rewards/margins": 4.009669303894043,
+      "rewards/rejected": -1.9071123600006104,
+      "step": 984
+    },
+    {
+      "epoch": 62.73,
+      "grad_norm": 1.6917866468429565,
+      "learning_rate": 3.6525612893355314e-07,
+      "logits/chosen": -2.564279317855835,
+      "logits/rejected": -2.569469928741455,
+      "logps/chosen": -109.41111755371094,
+      "logps/rejected": -116.52373504638672,
+      "loss": 0.0609,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.962429165840149,
+      "rewards/margins": 3.8329784870147705,
+      "rewards/rejected": -1.870548963546753,
+      "step": 988
+    },
+    {
+      "epoch": 62.98,
+      "grad_norm": 0.7290852069854736,
+      "learning_rate": 3.6425214558510574e-07,
+      "logits/chosen": -2.5594987869262695,
+      "logits/rejected": -2.583799362182617,
+      "logps/chosen": -109.19231414794922,
+      "logps/rejected": -115.69007110595703,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.7536541223526,
+      "rewards/margins": 3.967104434967041,
+      "rewards/rejected": -2.2134504318237305,
+      "step": 992
+    },
+    {
+      "epoch": 63.24,
+      "grad_norm": 0.9925839304924011,
+      "learning_rate": 3.6324582773728705e-07,
+      "logits/chosen": -2.5263614654541016,
+      "logits/rejected": -2.528245210647583,
+      "logps/chosen": -111.07906341552734,
+      "logps/rejected": -120.55313873291016,
+      "loss": 0.0504,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9851771593093872,
+      "rewards/margins": 3.987983465194702,
+      "rewards/rejected": -2.0028064250946045,
+      "step": 996
+    },
+    {
+      "epoch": 63.49,
+      "grad_norm": 0.9143023490905762,
+      "learning_rate": 3.622371959520599e-07,
+      "logits/chosen": -2.602895498275757,
+      "logits/rejected": -2.588486909866333,
+      "logps/chosen": -113.23171997070312,
+      "logps/rejected": -125.9736328125,
+      "loss": 0.0489,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9089276790618896,
+      "rewards/margins": 3.9534406661987305,
+      "rewards/rejected": -2.044512987136841,
+      "step": 1000
+    },
+    {
+      "epoch": 63.49,
+      "eval_logits/chosen": -2.6193482875823975,
+      "eval_logits/rejected": -2.669416666030884,
+      "eval_logps/chosen": -122.88963317871094,
+      "eval_logps/rejected": -109.84455108642578,
+      "eval_loss": 0.3812538683414459,
+      "eval_rewards/accuracies": 0.8214285969734192,
+      "eval_rewards/chosen": 1.1179710626602173,
+      "eval_rewards/margins": 2.291536808013916,
+      "eval_rewards/rejected": -1.1735659837722778,
+      "eval_runtime": 19.3806,
+      "eval_samples_per_second": 1.445,
+      "eval_steps_per_second": 1.445,
+      "step": 1000
+    },
+    {
+      "epoch": 63.75,
+      "grad_norm": 1.1189945936203003,
+      "learning_rate": 3.612262708386677e-07,
+      "logits/chosen": -2.549151659011841,
+      "logits/rejected": -2.5342297554016113,
+      "logps/chosen": -108.18368530273438,
+      "logps/rejected": -117.32633972167969,
+      "loss": 0.0552,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.9939894676208496,
+      "rewards/margins": 4.113208293914795,
+      "rewards/rejected": -2.1192190647125244,
+      "step": 1004
+    },
+    {
+      "epoch": 64.0,
+      "grad_norm": 0.9281061291694641,
+      "learning_rate": 3.6021307305321294e-07,
+      "logits/chosen": -2.524024724960327,
+      "logits/rejected": -2.5277915000915527,
+      "logps/chosen": -114.27344512939453,
+      "logps/rejected": -116.72073364257812,
+      "loss": 0.0341,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9564348459243774,
+      "rewards/margins": 4.206700325012207,
+      "rewards/rejected": -2.250265598297119,
+      "step": 1008
+    },
+    {
+      "epoch": 64.25,
+      "grad_norm": 0.9672576785087585,
+      "learning_rate": 3.591976232982355e-07,
+      "logits/chosen": -2.5507400035858154,
+      "logits/rejected": -2.5656113624572754,
+      "logps/chosen": -105.44967651367188,
+      "logps/rejected": -119.23912048339844,
+      "loss": 0.0448,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9570220708847046,
+      "rewards/margins": 4.062612533569336,
+      "rewards/rejected": -2.105590343475342,
+      "step": 1012
+    },
+    {
+      "epoch": 64.51,
+      "grad_norm": 0.9295753836631775,
+      "learning_rate": 3.581799423222894e-07,
+      "logits/chosen": -2.5452866554260254,
+      "logits/rejected": -2.5547432899475098,
+      "logps/chosen": -114.10334777832031,
+      "logps/rejected": -124.42996215820312,
+      "loss": 0.04,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8515616655349731,
+      "rewards/margins": 4.295123100280762,
+      "rewards/rejected": -2.443561315536499,
+      "step": 1016
+    },
+    {
+      "epoch": 64.76,
+      "grad_norm": 1.1630864143371582,
+      "learning_rate": 3.57160050919519e-07,
+      "logits/chosen": -2.5098414421081543,
+      "logits/rejected": -2.5110950469970703,
+      "logps/chosen": -112.0383529663086,
+      "logps/rejected": -125.51400756835938,
+      "loss": 0.0388,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0512752532958984,
+      "rewards/margins": 4.277214050292969,
+      "rewards/rejected": -2.2259387969970703,
+      "step": 1020
+    },
+    {
+      "epoch": 65.02,
+      "grad_norm": 1.1040410995483398,
+      "learning_rate": 3.561379699292338e-07,
+      "logits/chosen": -2.5779294967651367,
+      "logits/rejected": -2.532668113708496,
+      "logps/chosen": -114.37411499023438,
+      "logps/rejected": -112.6378173828125,
+      "loss": 0.0557,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 2.033304214477539,
+      "rewards/margins": 3.9332354068756104,
+      "rewards/rejected": -1.8999310731887817,
+      "step": 1024
+    },
+    {
+      "epoch": 65.27,
+      "grad_norm": 0.9148807525634766,
+      "learning_rate": 3.5511372023548305e-07,
+      "logits/chosen": -2.5077385902404785,
+      "logits/rejected": -2.5048418045043945,
+      "logps/chosen": -114.31764221191406,
+      "logps/rejected": -126.50031280517578,
+      "loss": 0.0509,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8525328636169434,
+      "rewards/margins": 4.272181987762451,
+      "rewards/rejected": -2.4196486473083496,
+      "step": 1028
+    },
+    {
+      "epoch": 65.52,
+      "grad_norm": 1.133243441581726,
+      "learning_rate": 3.5408732276662876e-07,
+      "logits/chosen": -2.5681333541870117,
+      "logits/rejected": -2.5366806983947754,
+      "logps/chosen": -108.5892333984375,
+      "logps/rejected": -117.7904281616211,
+      "loss": 0.0557,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.8624019622802734,
+      "rewards/margins": 3.896911144256592,
+      "rewards/rejected": -2.0345091819763184,
+      "step": 1032
+    },
+    {
+      "epoch": 65.78,
+      "grad_norm": 1.1898314952850342,
+      "learning_rate": 3.530587984949183e-07,
+      "logits/chosen": -2.5765511989593506,
+      "logits/rejected": -2.5598666667938232,
+      "logps/chosen": -116.97229766845703,
+      "logps/rejected": -121.18170166015625,
+      "loss": 0.0372,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0075888633728027,
+      "rewards/margins": 4.278448581695557,
+      "rewards/rejected": -2.270859718322754,
+      "step": 1036
+    },
+    {
+      "epoch": 66.03,
+      "grad_norm": 0.6972880363464355,
+      "learning_rate": 3.5202816843605535e-07,
+      "logits/chosen": -2.5401930809020996,
+      "logits/rejected": -2.5507655143737793,
+      "logps/chosen": -112.83836364746094,
+      "logps/rejected": -125.02684020996094,
+      "loss": 0.0293,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9683219194412231,
+      "rewards/margins": 4.259614944458008,
+      "rewards/rejected": -2.291292905807495,
+      "step": 1040
+    },
+    {
+      "epoch": 66.03,
+      "eval_logits/chosen": -2.607311725616455,
+      "eval_logits/rejected": -2.655458450317383,
+      "eval_logps/chosen": -123.29393005371094,
+      "eval_logps/rejected": -111.0044937133789,
+      "eval_loss": 0.38039258122444153,
+      "eval_rewards/accuracies": 0.8214285969734192,
+      "eval_rewards/chosen": 1.0775415897369385,
+      "eval_rewards/margins": 2.367100954055786,
+      "eval_rewards/rejected": -1.2895591259002686,
+      "eval_runtime": 19.3785,
+      "eval_samples_per_second": 1.445,
+      "eval_steps_per_second": 1.445,
+      "step": 1040
+    },
+    {
+      "epoch": 66.29,
+      "grad_norm": 1.1324729919433594,
+      "learning_rate": 3.5099545364877136e-07,
+      "logits/chosen": -2.540757417678833,
+      "logits/rejected": -2.5166916847229004,
+      "logps/chosen": -107.70198822021484,
+      "logps/rejected": -126.90846252441406,
+      "loss": 0.0483,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7595597505569458,
+      "rewards/margins": 3.971503257751465,
+      "rewards/rejected": -2.2119436264038086,
+      "step": 1044
+    },
+    {
+      "epoch": 66.54,
+      "grad_norm": 0.5052275061607361,
+      "learning_rate": 3.499606752343945e-07,
+      "logits/chosen": -2.5287301540374756,
+      "logits/rejected": -2.546327590942383,
+      "logps/chosen": -113.01952362060547,
+      "logps/rejected": -116.29533386230469,
+      "loss": 0.0356,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 2.0772006511688232,
+      "rewards/margins": 4.4480438232421875,
+      "rewards/rejected": -2.3708431720733643,
+      "step": 1048
+    },
+    {
+      "epoch": 66.79,
+      "grad_norm": 0.5736338496208191,
+      "learning_rate": 3.489238543364187e-07,
+      "logits/chosen": -2.5480170249938965,
+      "logits/rejected": -2.528210163116455,
+      "logps/chosen": -110.82701873779297,
+      "logps/rejected": -121.10733032226562,
+      "loss": 0.0264,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.2313339710235596,
+      "rewards/margins": 4.678577423095703,
+      "rewards/rejected": -2.4472432136535645,
+      "step": 1052
+    },
+    {
+      "epoch": 67.05,
+      "grad_norm": 0.7738077044487,
+      "learning_rate": 3.4788501214007186e-07,
+      "logits/chosen": -2.5314526557922363,
+      "logits/rejected": -2.513582468032837,
+      "logps/chosen": -116.080322265625,
+      "logps/rejected": -121.7528076171875,
+      "loss": 0.0543,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8950271606445312,
+      "rewards/margins": 4.078221797943115,
+      "rewards/rejected": -2.183194398880005,
+      "step": 1056
+    },
+    {
+      "epoch": 67.3,
+      "grad_norm": 0.618524432182312,
+      "learning_rate": 3.468441698718827e-07,
+      "logits/chosen": -2.559298276901245,
+      "logits/rejected": -2.551567554473877,
+      "logps/chosen": -111.18084716796875,
+      "logps/rejected": -122.01901245117188,
+      "loss": 0.0331,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9931435585021973,
+      "rewards/margins": 4.283310413360596,
+      "rewards/rejected": -2.2901663780212402,
+      "step": 1060
+    },
+    {
+      "epoch": 67.56,
+      "grad_norm": 0.9859129786491394,
+      "learning_rate": 3.4580134879924727e-07,
+      "logits/chosen": -2.5489492416381836,
+      "logits/rejected": -2.5233051776885986,
+      "logps/chosen": -114.399658203125,
+      "logps/rejected": -118.5413818359375,
+      "loss": 0.0444,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.026573896408081,
+      "rewards/margins": 4.122701644897461,
+      "rewards/rejected": -2.096128225326538,
+      "step": 1064
+    },
+    {
+      "epoch": 67.81,
+      "grad_norm": 0.8273893594741821,
+      "learning_rate": 3.4475657022999414e-07,
+      "logits/chosen": -2.5420095920562744,
+      "logits/rejected": -2.554413318634033,
+      "logps/chosen": -111.864501953125,
+      "logps/rejected": -121.59029388427734,
+      "loss": 0.0398,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.890322208404541,
+      "rewards/margins": 4.2500786781311035,
+      "rewards/rejected": -2.3597564697265625,
+      "step": 1068
+    },
+    {
+      "epoch": 68.06,
+      "grad_norm": 0.8102222084999084,
+      "learning_rate": 3.4370985551194926e-07,
+      "logits/chosen": -2.489243984222412,
+      "logits/rejected": -2.492791175842285,
+      "logps/chosen": -105.2918701171875,
+      "logps/rejected": -127.78358459472656,
+      "loss": 0.0428,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.985128402709961,
+      "rewards/margins": 4.529362678527832,
+      "rewards/rejected": -2.544234275817871,
+      "step": 1072
+    },
+    {
+      "epoch": 68.32,
+      "grad_norm": 0.8171394467353821,
+      "learning_rate": 3.426612260324995e-07,
+      "logits/chosen": -2.5788776874542236,
+      "logits/rejected": -2.5668318271636963,
+      "logps/chosen": -113.4781265258789,
+      "logps/rejected": -119.6295166015625,
+      "loss": 0.031,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9655818939208984,
+      "rewards/margins": 4.498297214508057,
+      "rewards/rejected": -2.5327155590057373,
+      "step": 1076
+    },
+    {
+      "epoch": 68.57,
+      "grad_norm": 0.6531574130058289,
+      "learning_rate": 3.41610703218156e-07,
+      "logits/chosen": -2.5451254844665527,
+      "logits/rejected": -2.539482593536377,
+      "logps/chosen": -111.55757141113281,
+      "logps/rejected": -122.69400787353516,
+      "loss": 0.038,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8812763690948486,
+      "rewards/margins": 4.159421920776367,
+      "rewards/rejected": -2.2781457901000977,
+      "step": 1080
+    },
+    {
+      "epoch": 68.57,
+      "eval_logits/chosen": -2.595308542251587,
+      "eval_logits/rejected": -2.6415059566497803,
+      "eval_logps/chosen": -123.70062255859375,
+      "eval_logps/rejected": -112.08638763427734,
+      "eval_loss": 0.38112249970436096,
+      "eval_rewards/accuracies": 0.8214285969734192,
+      "eval_rewards/chosen": 1.0368707180023193,
+      "eval_rewards/margins": 2.4346213340759277,
+      "eval_rewards/rejected": -1.3977504968643188,
+      "eval_runtime": 19.362,
+      "eval_samples_per_second": 1.446,
+      "eval_steps_per_second": 1.446,
+      "step": 1080
+    },
+    {
+      "epoch": 68.83,
+      "grad_norm": 0.43723320960998535,
+      "learning_rate": 3.405583085341161e-07,
+      "logits/chosen": -2.4995832443237305,
+      "logits/rejected": -2.4867968559265137,
+      "logps/chosen": -113.4669418334961,
+      "logps/rejected": -126.17251586914062,
+      "loss": 0.0472,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.8598037958145142,
+      "rewards/margins": 4.452800750732422,
+      "rewards/rejected": -2.5929970741271973,
+      "step": 1084
+    },
+    {
+      "epoch": 69.08,
+      "grad_norm": 0.6834803819656372,
+      "learning_rate": 3.3950406348382475e-07,
+      "logits/chosen": -2.5350327491760254,
+      "logits/rejected": -2.523555040359497,
+      "logps/chosen": -109.96227264404297,
+      "logps/rejected": -127.48822021484375,
+      "loss": 0.0343,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8542053699493408,
+      "rewards/margins": 4.349175453186035,
+      "rewards/rejected": -2.4949703216552734,
+      "step": 1088
+    },
+    {
+      "epoch": 69.33,
+      "grad_norm": 0.9091951251029968,
+      "learning_rate": 3.384479896085353e-07,
+      "logits/chosen": -2.5270514488220215,
+      "logits/rejected": -2.518280267715454,
+      "logps/chosen": -112.04927825927734,
+      "logps/rejected": -118.20771789550781,
+      "loss": 0.0355,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.085174322128296,
+      "rewards/margins": 4.528075218200684,
+      "rewards/rejected": -2.442901134490967,
+      "step": 1092
+    },
+    {
+      "epoch": 69.59,
+      "grad_norm": 0.7112252116203308,
+      "learning_rate": 3.3739010848686903e-07,
+      "logits/chosen": -2.5208559036254883,
+      "logits/rejected": -2.518951177597046,
+      "logps/chosen": -114.5992431640625,
+      "logps/rejected": -129.33493041992188,
+      "loss": 0.029,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0512499809265137,
+      "rewards/margins": 4.543766498565674,
+      "rewards/rejected": -2.492516279220581,
+      "step": 1096
+    },
+    {
+      "epoch": 69.84,
+      "grad_norm": 0.948445200920105,
+      "learning_rate": 3.3633044173437487e-07,
+      "logits/chosen": -2.53946590423584,
+      "logits/rejected": -2.5132739543914795,
+      "logps/chosen": -113.12407684326172,
+      "logps/rejected": -120.42703247070312,
+      "loss": 0.0376,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8349709510803223,
+      "rewards/margins": 4.3107500076293945,
+      "rewards/rejected": -2.4757792949676514,
+      "step": 1100
+    },
+    {
+      "epoch": 70.1,
+      "grad_norm": 1.1577597856521606,
+      "learning_rate": 3.352690110030869e-07,
+      "logits/chosen": -2.5429604053497314,
+      "logits/rejected": -2.5576508045196533,
+      "logps/chosen": -107.9112548828125,
+      "logps/rejected": -126.51514434814453,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.8008710145950317,
+      "rewards/margins": 4.010225772857666,
+      "rewards/rejected": -2.2093546390533447,
+      "step": 1104
+    },
+    {
+      "epoch": 70.35,
+      "grad_norm": 0.8540000915527344,
+      "learning_rate": 3.342058379810825e-07,
+      "logits/chosen": -2.5239222049713135,
+      "logits/rejected": -2.508244037628174,
+      "logps/chosen": -111.51808166503906,
+      "logps/rejected": -127.96340942382812,
+      "loss": 0.039,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.8233864307403564,
+      "rewards/margins": 4.78690242767334,
+      "rewards/rejected": -2.9635162353515625,
+      "step": 1108
+    },
+    {
+      "epoch": 70.6,
+      "grad_norm": 0.8324258923530579,
+      "learning_rate": 3.33140944392039e-07,
+      "logits/chosen": -2.560696601867676,
+      "logits/rejected": -2.5394399166107178,
+      "logps/chosen": -104.02289581298828,
+      "logps/rejected": -118.75849914550781,
+      "loss": 0.0335,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.004833698272705,
+      "rewards/margins": 4.301353931427002,
+      "rewards/rejected": -2.2965199947357178,
+      "step": 1112
+    },
+    {
+      "epoch": 70.86,
+      "grad_norm": 0.9126656651496887,
+      "learning_rate": 3.3207435199479005e-07,
+      "logits/chosen": -2.495453119277954,
+      "logits/rejected": -2.476893424987793,
+      "logps/chosen": -124.06426239013672,
+      "logps/rejected": -126.9112777709961,
+      "loss": 0.0275,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0254414081573486,
+      "rewards/margins": 4.661925315856934,
+      "rewards/rejected": -2.636483669281006,
+      "step": 1116
+    },
+    {
+      "epoch": 71.11,
+      "grad_norm": 0.7946174740791321,
+      "learning_rate": 3.3100608258288067e-07,
+      "logits/chosen": -2.558216094970703,
+      "logits/rejected": -2.5525529384613037,
+      "logps/chosen": -109.46134948730469,
+      "logps/rejected": -122.28605651855469,
+      "loss": 0.0378,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9690169095993042,
+      "rewards/margins": 4.215582370758057,
+      "rewards/rejected": -2.246565580368042,
+      "step": 1120
+    },
+    {
+      "epoch": 71.11,
+      "eval_logits/chosen": -2.5836751461029053,
+      "eval_logits/rejected": -2.6279237270355225,
+      "eval_logps/chosen": -124.22281646728516,
+      "eval_logps/rejected": -113.24488067626953,
+      "eval_loss": 0.3797485828399658,
+      "eval_rewards/accuracies": 0.8214285969734192,
+      "eval_rewards/chosen": 0.9846526384353638,
+      "eval_rewards/margins": 2.4982528686523438,
+      "eval_rewards/rejected": -1.51360023021698,
+      "eval_runtime": 19.3727,
+      "eval_samples_per_second": 1.445,
+      "eval_steps_per_second": 1.445,
+      "step": 1120
+    },
+    {
+      "epoch": 71.37,
+      "grad_norm": 0.8299816846847534,
+      "learning_rate": 3.29936157984122e-07,
+      "logits/chosen": -2.49894118309021,
+      "logits/rejected": -2.514620304107666,
+      "logps/chosen": -113.71720886230469,
+      "logps/rejected": -125.64193725585938,
+      "loss": 0.0336,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9318768978118896,
+      "rewards/margins": 4.486914157867432,
+      "rewards/rejected": -2.555037021636963,
+      "step": 1124
+    },
+    {
+      "epoch": 71.62,
+      "grad_norm": 1.118394136428833,
+      "learning_rate": 3.2886460006014564e-07,
+      "logits/chosen": -2.5559606552124023,
+      "logits/rejected": -2.531343936920166,
+      "logps/chosen": -115.6338119506836,
+      "logps/rejected": -127.78579711914062,
+      "loss": 0.0327,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.0132534503936768,
+      "rewards/margins": 4.542009353637695,
+      "rewards/rejected": -2.5287556648254395,
+      "step": 1128
+    },
+    {
+      "epoch": 71.87,
+      "grad_norm": 0.7830736041069031,
+      "learning_rate": 3.2779143070595654e-07,
+      "logits/chosen": -2.500929594039917,
+      "logits/rejected": -2.489039897918701,
+      "logps/chosen": -109.16018676757812,
+      "logps/rejected": -119.57136535644531,
+      "loss": 0.0369,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.957329511642456,
+      "rewards/margins": 4.499155521392822,
+      "rewards/rejected": -2.541826009750366,
+      "step": 1132
+    },
+    {
+      "epoch": 72.13,
+      "grad_norm": 0.8632897138595581,
+      "learning_rate": 3.2671667184948606e-07,
+      "logits/chosen": -2.499505043029785,
+      "logits/rejected": -2.479982852935791,
+      "logps/chosen": -113.5299072265625,
+      "logps/rejected": -124.4341049194336,
+      "loss": 0.0272,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7402851581573486,
+      "rewards/margins": 4.627835750579834,
+      "rewards/rejected": -2.8875510692596436,
+      "step": 1136
+    },
+    {
+      "epoch": 72.38,
+      "grad_norm": 0.5583193302154541,
+      "learning_rate": 3.2564034545114307e-07,
+      "logits/chosen": -2.5371761322021484,
+      "logits/rejected": -2.5350067615509033,
+      "logps/chosen": -113.26782989501953,
+      "logps/rejected": -123.95820617675781,
+      "loss": 0.0311,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.9537227153778076,
+      "rewards/margins": 4.536089897155762,
+      "rewards/rejected": -2.582366943359375,
+      "step": 1140
+    },
+    {
+      "epoch": 72.63,
+      "grad_norm": 0.7022975087165833,
+      "learning_rate": 3.2456247350336644e-07,
+      "logits/chosen": -2.4975104331970215,
+      "logits/rejected": -2.4862759113311768,
+      "logps/chosen": -116.6255111694336,
+      "logps/rejected": -124.94264221191406,
+      "loss": 0.0405,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.032803535461426,
+      "rewards/margins": 4.609248161315918,
+      "rewards/rejected": -2.576444387435913,
+      "step": 1144
+    },
+    {
+      "epoch": 72.89,
+      "grad_norm": 0.6374495625495911,
+      "learning_rate": 3.234830780301749e-07,
+      "logits/chosen": -2.5176424980163574,
+      "logits/rejected": -2.5058205127716064,
+      "logps/chosen": -103.94803619384766,
+      "logps/rejected": -128.84860229492188,
+      "loss": 0.0329,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 1.949141502380371,
+      "rewards/margins": 4.762382507324219,
+      "rewards/rejected": -2.8132412433624268,
+      "step": 1148
+    },
+    {
+      "epoch": 73.14,
+      "grad_norm": 0.6864801049232483,
+      "learning_rate": 3.224021810867168e-07,
+      "logits/chosen": -2.519915819168091,
+      "logits/rejected": -2.5309557914733887,
+      "logps/chosen": -112.75774383544922,
+      "logps/rejected": -125.47956085205078,
+      "loss": 0.0333,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8436543941497803,
+      "rewards/margins": 4.395434856414795,
+      "rewards/rejected": -2.5517804622650146,
+      "step": 1152
+    },
+    {
+      "epoch": 73.4,
+      "grad_norm": 0.8171544075012207,
+      "learning_rate": 3.213198047588205e-07,
+      "logits/chosen": -2.529101610183716,
+      "logits/rejected": -2.506753921508789,
+      "logps/chosen": -111.0318603515625,
+      "logps/rejected": -128.049072265625,
+      "loss": 0.0335,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7638250589370728,
+      "rewards/margins": 4.518660068511963,
+      "rewards/rejected": -2.7548348903656006,
+      "step": 1156
+    },
+    {
+      "epoch": 73.65,
+      "grad_norm": 0.7904006838798523,
+      "learning_rate": 3.202359711625417e-07,
+      "logits/chosen": -2.507455348968506,
+      "logits/rejected": -2.48976993560791,
+      "logps/chosen": -113.42962646484375,
+      "logps/rejected": -128.28025817871094,
+      "loss": 0.0288,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8584176301956177,
+      "rewards/margins": 4.5566816329956055,
+      "rewards/rejected": -2.6982643604278564,
+      "step": 1160
+    },
+    {
+      "epoch": 73.65,
+      "eval_logits/chosen": -2.5730206966400146,
+      "eval_logits/rejected": -2.615673303604126,
+      "eval_logps/chosen": -124.67210388183594,
+      "eval_logps/rejected": -114.29837799072266,
+      "eval_loss": 0.38018837571144104,
+      "eval_rewards/accuracies": 0.8214285969734192,
+      "eval_rewards/chosen": 0.939724326133728,
+      "eval_rewards/margins": 2.558673143386841,
+      "eval_rewards/rejected": -1.6189485788345337,
+      "eval_runtime": 19.4065,
+      "eval_samples_per_second": 1.443,
+      "eval_steps_per_second": 1.443,
+      "step": 1160
+    }
+  ],
+  "logging_steps": 4,
+  "max_steps": 2800,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 187,
+  "save_steps": 40,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}