{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 35.55555555555556,
  "eval_steps": 40,
  "global_step": 560,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.25,
      "grad_norm": 7.928347587585449,
      "learning_rate": 7.5e-08,
      "logits/chosen": -2.7277705669403076,
      "logits/rejected": -2.7679762840270996,
      "logps/chosen": -128.4798583984375,
      "logps/rejected": -98.36178588867188,
      "loss": 0.6931,
      "rewards/accuracies": 0.109375,
      "rewards/chosen": -7.17043731128797e-05,
      "rewards/margins": 5.325676465872675e-05,
      "rewards/rejected": -0.0001249611668754369,
      "step": 4
    },
    {
      "epoch": 0.51,
      "grad_norm": 9.242063522338867,
      "learning_rate": 1.75e-07,
      "logits/chosen": -2.6805009841918945,
      "logits/rejected": -2.712625026702881,
      "logps/chosen": -138.11959838867188,
      "logps/rejected": -96.05882263183594,
      "loss": 0.693,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.0008271098486147821,
      "rewards/margins": 0.00034989125560969114,
      "rewards/rejected": 0.00047721865121275187,
      "step": 8
    },
    {
      "epoch": 0.76,
      "grad_norm": 8.156970977783203,
      "learning_rate": 2.75e-07,
      "logits/chosen": -2.7695400714874268,
      "logits/rejected": -2.7420883178710938,
      "logps/chosen": -133.04307556152344,
      "logps/rejected": -104.19400024414062,
      "loss": 0.6919,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.0034509659744799137,
      "rewards/margins": 0.0025641382671892643,
      "rewards/rejected": 0.0008868275908753276,
      "step": 12
    },
    {
      "epoch": 1.02,
      "grad_norm": 8.723258018493652,
      "learning_rate": 3.75e-07,
      "logits/chosen": -2.6869003772735596,
      "logits/rejected": -2.7081785202026367,
      "logps/chosen": -126.2979736328125,
      "logps/rejected": -98.4257583618164,
      "loss": 0.6889,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.011365305632352829,
      "rewards/margins": 0.008443659171462059,
      "rewards/rejected": 0.0029216469265520573,
      "step": 16
    },
    {
      "epoch": 1.27,
      "grad_norm": 10.34566593170166,
      "learning_rate": 4.7499999999999995e-07,
      "logits/chosen": -2.704402208328247,
      "logits/rejected": -2.764737367630005,
      "logps/chosen": -125.91941833496094,
      "logps/rejected": -99.02344512939453,
      "loss": 0.6832,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.02058127522468567,
      "rewards/margins": 0.020110297948122025,
      "rewards/rejected": 0.00047097797505557537,
      "step": 20
    },
    {
      "epoch": 1.52,
      "grad_norm": 9.291991233825684,
      "learning_rate": 4.999985633126757e-07,
      "logits/chosen": -2.716165781021118,
      "logits/rejected": -2.6845083236694336,
      "logps/chosen": -137.2718505859375,
      "logps/rejected": -97.50873565673828,
      "loss": 0.6785,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.03298673778772354,
      "rewards/margins": 0.029706846922636032,
      "rewards/rejected": 0.0032798887696117163,
      "step": 24
    },
    {
      "epoch": 1.78,
      "grad_norm": 9.161444664001465,
      "learning_rate": 4.999921780689761e-07,
      "logits/chosen": -2.735781192779541,
      "logits/rejected": -2.7485365867614746,
      "logps/chosen": -128.5098876953125,
      "logps/rejected": -90.1419906616211,
      "loss": 0.6693,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.04758748784661293,
      "rewards/margins": 0.04847026616334915,
      "rewards/rejected": -0.0008827749406918883,
      "step": 28
    },
    {
      "epoch": 2.03,
      "grad_norm": 8.495955467224121,
      "learning_rate": 4.99980684767309e-07,
      "logits/chosen": -2.7206833362579346,
      "logits/rejected": -2.750042676925659,
      "logps/chosen": -133.54827880859375,
      "logps/rejected": -108.23342895507812,
      "loss": 0.6669,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.0631466805934906,
      "rewards/margins": 0.05378420650959015,
      "rewards/rejected": 0.009362474083900452,
      "step": 32
    },
    {
      "epoch": 2.29,
      "grad_norm": 8.951277732849121,
      "learning_rate": 4.999640836425158e-07,
      "logits/chosen": -2.6706807613372803,
      "logits/rejected": -2.6151680946350098,
      "logps/chosen": -136.55226135253906,
      "logps/rejected": -97.76315307617188,
      "loss": 0.6578,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.08149851858615875,
      "rewards/margins": 0.07302143424749374,
      "rewards/rejected": 0.008477086201310158,
      "step": 36
    },
    {
      "epoch": 2.54,
      "grad_norm": 8.608317375183105,
      "learning_rate": 4.99942375033805e-07,
      "logits/chosen": -2.7676639556884766,
      "logits/rejected": -2.833590030670166,
      "logps/chosen": -133.7703857421875,
      "logps/rejected": -106.12245178222656,
      "loss": 0.6488,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10222765803337097,
      "rewards/margins": 0.09170522540807724,
      "rewards/rejected": 0.010522443801164627,
      "step": 40
    },
    {
      "epoch": 2.54,
      "eval_logits/chosen": -2.806352138519287,
      "eval_logits/rejected": -2.8898861408233643,
      "eval_logps/chosen": -133.14093017578125,
      "eval_logps/rejected": -97.8843002319336,
      "eval_loss": 0.6592782139778137,
      "eval_rewards/accuracies": 0.7857142686843872,
      "eval_rewards/chosen": 0.09283973276615143,
      "eval_rewards/margins": 0.07038124650716782,
      "eval_rewards/rejected": 0.022458484396338463,
      "eval_runtime": 19.3683,
      "eval_samples_per_second": 1.446,
      "eval_steps_per_second": 1.446,
      "step": 40
    },
    {
      "epoch": 2.79,
      "grad_norm": 9.09485912322998,
      "learning_rate": 4.999155593847457e-07,
      "logits/chosen": -2.7024385929107666,
      "logits/rejected": -2.7564737796783447,
      "logps/chosen": -123.412353515625,
      "logps/rejected": -91.91966247558594,
      "loss": 0.6347,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.114817313849926,
      "rewards/margins": 0.12200291454792023,
      "rewards/rejected": -0.007185595575720072,
      "step": 44
    },
    {
      "epoch": 3.05,
      "grad_norm": 9.453149795532227,
      "learning_rate": 4.998836372432589e-07,
      "logits/chosen": -2.736665725708008,
      "logits/rejected": -2.757841110229492,
      "logps/chosen": -127.0240478515625,
      "logps/rejected": -103.66780853271484,
      "loss": 0.636,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.13144327700138092,
      "rewards/margins": 0.11972987651824951,
      "rewards/rejected": 0.011713397689163685,
      "step": 48
    },
    {
      "epoch": 3.3,
      "grad_norm": 8.674492835998535,
      "learning_rate": 4.998466092616052e-07,
      "logits/chosen": -2.710110664367676,
      "logits/rejected": -2.7205581665039062,
      "logps/chosen": -131.0464324951172,
      "logps/rejected": -99.71279907226562,
      "loss": 0.6166,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.15897569060325623,
      "rewards/margins": 0.16229701042175293,
      "rewards/rejected": -0.003321333322674036,
      "step": 52
    },
    {
      "epoch": 3.56,
      "grad_norm": 9.434328079223633,
      "learning_rate": 4.99804476196373e-07,
      "logits/chosen": -2.685814380645752,
      "logits/rejected": -2.736269474029541,
      "logps/chosen": -125.750732421875,
      "logps/rejected": -93.69428253173828,
      "loss": 0.6131,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.16781488060951233,
      "rewards/margins": 0.16998659074306488,
      "rewards/rejected": -0.002171688713133335,
      "step": 56
    },
    {
      "epoch": 3.81,
      "grad_norm": 8.643455505371094,
      "learning_rate": 4.997572389084615e-07,
      "logits/chosen": -2.7429587841033936,
      "logits/rejected": -2.75490140914917,
      "logps/chosen": -131.4405517578125,
      "logps/rejected": -99.93754577636719,
      "loss": 0.614,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.18873561918735504,
      "rewards/margins": 0.17132429778575897,
      "rewards/rejected": 0.01741132140159607,
      "step": 60
    },
    {
      "epoch": 4.06,
      "grad_norm": 8.412778854370117,
      "learning_rate": 4.997048983630643e-07,
      "logits/chosen": -2.6886143684387207,
      "logits/rejected": -2.710732936859131,
      "logps/chosen": -123.77167510986328,
      "logps/rejected": -97.30846405029297,
      "loss": 0.6099,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19313779473304749,
      "rewards/margins": 0.1793561577796936,
      "rewards/rejected": 0.013781649991869926,
      "step": 64
    },
    {
      "epoch": 4.32,
      "grad_norm": 7.851288795471191,
      "learning_rate": 4.99647455629649e-07,
      "logits/chosen": -2.744528293609619,
      "logits/rejected": -2.7117881774902344,
      "logps/chosen": -136.25863647460938,
      "logps/rejected": -95.11305236816406,
      "loss": 0.5968,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21463464200496674,
      "rewards/margins": 0.20793242752552032,
      "rewards/rejected": 0.0067022331058979034,
      "step": 68
    },
    {
      "epoch": 4.57,
      "grad_norm": 9.278959274291992,
      "learning_rate": 4.995849118819353e-07,
      "logits/chosen": -2.7611894607543945,
      "logits/rejected": -2.7447633743286133,
      "logps/chosen": -134.3462677001953,
      "logps/rejected": -103.75216674804688,
      "loss": 0.5828,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.25433310866355896,
      "rewards/margins": 0.24298511445522308,
      "rewards/rejected": 0.011347985826432705,
      "step": 72
    },
    {
      "epoch": 4.83,
      "grad_norm": 7.795536994934082,
      "learning_rate": 4.995172683978719e-07,
      "logits/chosen": -2.698429584503174,
      "logits/rejected": -2.7458348274230957,
      "logps/chosen": -127.65770721435547,
      "logps/rejected": -104.23495483398438,
      "loss": 0.5796,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.25434932112693787,
      "rewards/margins": 0.25114190578460693,
      "rewards/rejected": 0.0032073920592665672,
      "step": 76
    },
    {
      "epoch": 5.08,
      "grad_norm": 7.788575649261475,
      "learning_rate": 4.994445265596091e-07,
      "logits/chosen": -2.6734955310821533,
      "logits/rejected": -2.7292165756225586,
      "logps/chosen": -123.53501892089844,
      "logps/rejected": -95.23420715332031,
      "loss": 0.5676,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2950035631656647,
      "rewards/margins": 0.2805778682231903,
      "rewards/rejected": 0.014425676316022873,
      "step": 80
    },
    {
      "epoch": 5.08,
      "eval_logits/chosen": -2.8074097633361816,
      "eval_logits/rejected": -2.890533685684204,
      "eval_logps/chosen": -131.55104064941406,
      "eval_logps/rejected": -97.56663513183594,
      "eval_loss": 0.6045485734939575,
      "eval_rewards/accuracies": 0.7857142686843872,
      "eval_rewards/chosen": 0.2518289387226105,
      "eval_rewards/margins": 0.1976027488708496,
      "eval_rewards/rejected": 0.05422618240118027,
      "eval_runtime": 19.3586,
      "eval_samples_per_second": 1.446,
      "eval_steps_per_second": 1.446,
      "step": 80
    },
    {
      "epoch": 5.33,
      "grad_norm": 6.709670543670654,
      "learning_rate": 4.993666878534718e-07,
      "logits/chosen": -2.736870527267456,
      "logits/rejected": -2.778140068054199,
      "logps/chosen": -128.86972045898438,
      "logps/rejected": -104.70977020263672,
      "loss": 0.5637,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.31255263090133667,
      "rewards/margins": 0.2908375561237335,
      "rewards/rejected": 0.021715058013796806,
      "step": 84
    },
    {
      "epoch": 5.59,
      "grad_norm": 8.178019523620605,
      "learning_rate": 4.99283753869928e-07,
      "logits/chosen": -2.686039447784424,
      "logits/rejected": -2.681716203689575,
      "logps/chosen": -129.52798461914062,
      "logps/rejected": -97.80433654785156,
      "loss": 0.5475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32084640860557556,
      "rewards/margins": 0.33009591698646545,
      "rewards/rejected": -0.00924946740269661,
      "step": 88
    },
    {
      "epoch": 5.84,
      "grad_norm": 7.515178680419922,
      "learning_rate": 4.991957263035573e-07,
      "logits/chosen": -2.763765811920166,
      "logits/rejected": -2.7660324573516846,
      "logps/chosen": -128.16851806640625,
      "logps/rejected": -95.72579956054688,
      "loss": 0.5493,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3080848455429077,
      "rewards/margins": 0.32350555062294006,
      "rewards/rejected": -0.015420722775161266,
      "step": 92
    },
    {
      "epoch": 6.1,
      "grad_norm": 8.599517822265625,
      "learning_rate": 4.991026069530156e-07,
      "logits/chosen": -2.7107338905334473,
      "logits/rejected": -2.745981216430664,
      "logps/chosen": -131.5907745361328,
      "logps/rejected": -102.65729522705078,
      "loss": 0.5568,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.36698901653289795,
      "rewards/margins": 0.3141450881958008,
      "rewards/rejected": 0.05284389108419418,
      "step": 96
    },
    {
      "epoch": 6.35,
      "grad_norm": 7.087887287139893,
      "learning_rate": 4.990043977209984e-07,
      "logits/chosen": -2.75876522064209,
      "logits/rejected": -2.800281047821045,
      "logps/chosen": -124.87574005126953,
      "logps/rejected": -99.60267639160156,
      "loss": 0.5332,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3760470151901245,
      "rewards/margins": 0.37043848633766174,
      "rewards/rejected": 0.00560858054086566,
      "step": 100
    },
    {
      "epoch": 6.6,
      "grad_norm": 6.458583831787109,
      "learning_rate": 4.989011006142024e-07,
      "logits/chosen": -2.70744252204895,
      "logits/rejected": -2.7285239696502686,
      "logps/chosen": -129.41262817382812,
      "logps/rejected": -98.60986328125,
      "loss": 0.5286,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.3909401297569275,
      "rewards/margins": 0.3845981955528259,
      "rewards/rejected": 0.006341893225908279,
      "step": 104
    },
    {
      "epoch": 6.86,
      "grad_norm": 8.084393501281738,
      "learning_rate": 4.987927177432842e-07,
      "logits/chosen": -2.731127977371216,
      "logits/rejected": -2.7352988719940186,
      "logps/chosen": -123.85149383544922,
      "logps/rejected": -97.99423217773438,
      "loss": 0.544,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.38825392723083496,
      "rewards/margins": 0.34452566504478455,
      "rewards/rejected": 0.043728262186050415,
      "step": 108
    },
    {
      "epoch": 7.11,
      "grad_norm": 7.936505317687988,
      "learning_rate": 4.987080943856886e-07,
      "logits/chosen": -2.6931982040405273,
      "logits/rejected": -2.683288812637329,
      "logps/chosen": -130.97227478027344,
      "logps/rejected": -93.71176147460938,
      "loss": 0.4903,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.45238468050956726,
      "rewards/margins": 0.48204928636550903,
      "rewards/rejected": -0.029664600268006325,
      "step": 112
    },
    {
      "epoch": 7.37,
      "grad_norm": 6.827272415161133,
      "learning_rate": 4.985908168188602e-07,
      "logits/chosen": -2.697653293609619,
      "logits/rejected": -2.7467117309570312,
      "logps/chosen": -119.16637420654297,
      "logps/rejected": -95.0647201538086,
      "loss": 0.5088,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.4183903634548187,
      "rewards/margins": 0.43436184525489807,
      "rewards/rejected": -0.015971507877111435,
      "step": 116
    },
    {
      "epoch": 7.62,
      "grad_norm": 6.599144458770752,
      "learning_rate": 4.984684598278982e-07,
      "logits/chosen": -2.7136449813842773,
      "logits/rejected": -2.7430026531219482,
      "logps/chosen": -126.032958984375,
      "logps/rejected": -102.69947814941406,
      "loss": 0.5231,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4575308859348297,
      "rewards/margins": 0.40544256567955017,
      "rewards/rejected": 0.052088312804698944,
      "step": 120
    },
    {
      "epoch": 7.62,
      "eval_logits/chosen": -2.807668447494507,
      "eval_logits/rejected": -2.8901402950286865,
      "eval_logps/chosen": -129.97122192382812,
      "eval_logps/rejected": -97.28166961669922,
      "eval_loss": 0.5570675134658813,
      "eval_rewards/accuracies": 0.7857142686843872,
      "eval_rewards/chosen": 0.4098115563392639,
      "eval_rewards/margins": 0.3270883560180664,
      "eval_rewards/rejected": 0.0827232226729393,
      "eval_runtime": 19.4265,
      "eval_samples_per_second": 1.441,
      "eval_steps_per_second": 1.441,
      "step": 120
    },
    {
      "epoch": 7.87,
      "grad_norm": 7.769857883453369,
      "learning_rate": 4.983410259129073e-07,
      "logits/chosen": -2.7467093467712402,
      "logits/rejected": -2.7454986572265625,
      "logps/chosen": -134.60739135742188,
      "logps/rejected": -103.20536804199219,
      "loss": 0.4995,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.4985056221485138,
      "rewards/margins": 0.4699307978153229,
      "rewards/rejected": 0.028574790805578232,
      "step": 124
    },
    {
      "epoch": 8.13,
      "grad_norm": 7.224113941192627,
      "learning_rate": 4.982085176777285e-07,
      "logits/chosen": -2.725543975830078,
      "logits/rejected": -2.7536404132843018,
      "logps/chosen": -121.35929870605469,
      "logps/rejected": -97.49042510986328,
      "loss": 0.4988,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.4724448025226593,
      "rewards/margins": 0.4719679057598114,
      "rewards/rejected": 0.0004768962971866131,
      "step": 128
    },
    {
      "epoch": 8.38,
      "grad_norm": 5.916596412658691,
      "learning_rate": 4.980709378298851e-07,
      "logits/chosen": -2.7414817810058594,
      "logits/rejected": -2.7378463745117188,
      "logps/chosen": -119.76469421386719,
      "logps/rejected": -92.40216827392578,
      "loss": 0.4946,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.48038381338119507,
      "rewards/margins": 0.4809207320213318,
      "rewards/rejected": -0.0005369335412979126,
      "step": 132
    },
    {
      "epoch": 8.63,
      "grad_norm": 6.414724826812744,
      "learning_rate": 4.979282891805286e-07,
      "logits/chosen": -2.7029452323913574,
      "logits/rejected": -2.737753391265869,
      "logps/chosen": -130.59829711914062,
      "logps/rejected": -99.5873794555664,
      "loss": 0.4564,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.5609064698219299,
      "rewards/margins": 0.5927521586418152,
      "rewards/rejected": -0.03184571862220764,
      "step": 136
    },
    {
      "epoch": 8.89,
      "grad_norm": 7.006284236907959,
      "learning_rate": 4.977805746443806e-07,
      "logits/chosen": -2.719618320465088,
      "logits/rejected": -2.7250468730926514,
      "logps/chosen": -128.04678344726562,
      "logps/rejected": -101.81006622314453,
      "loss": 0.4761,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.5477940440177917,
      "rewards/margins": 0.5303957462310791,
      "rewards/rejected": 0.017398254945874214,
      "step": 140
    },
    {
      "epoch": 9.14,
      "grad_norm": 7.040762901306152,
      "learning_rate": 4.97627797239673e-07,
      "logits/chosen": -2.6895217895507812,
      "logits/rejected": -2.7162435054779053,
      "logps/chosen": -133.3381805419922,
      "logps/rejected": -102.25975799560547,
      "loss": 0.4403,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.6185128688812256,
      "rewards/margins": 0.6498669981956482,
      "rewards/rejected": -0.03135409206151962,
      "step": 144
    },
    {
      "epoch": 9.4,
      "grad_norm": 6.93245267868042,
      "learning_rate": 4.974699600880869e-07,
      "logits/chosen": -2.6911263465881348,
      "logits/rejected": -2.7075552940368652,
      "logps/chosen": -126.95785522460938,
      "logps/rejected": -102.03314971923828,
      "loss": 0.4587,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.573100745677948,
      "rewards/margins": 0.5804403424263,
      "rewards/rejected": -0.0073395660147070885,
      "step": 148
    },
    {
      "epoch": 9.65,
      "grad_norm": 7.084275245666504,
      "learning_rate": 4.973070664146885e-07,
      "logits/chosen": -2.7053210735321045,
      "logits/rejected": -2.7612156867980957,
      "logps/chosen": -119.92796325683594,
      "logps/rejected": -97.49836730957031,
      "loss": 0.483,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.5761281251907349,
      "rewards/margins": 0.5190825462341309,
      "rewards/rejected": 0.057045597583055496,
      "step": 152
    },
    {
      "epoch": 9.9,
      "grad_norm": 5.3888258934021,
      "learning_rate": 4.971391195478631e-07,
      "logits/chosen": -2.750107765197754,
      "logits/rejected": -2.7442452907562256,
      "logps/chosen": -128.85630798339844,
      "logps/rejected": -96.89845275878906,
      "loss": 0.4517,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6420415043830872,
      "rewards/margins": 0.6197397708892822,
      "rewards/rejected": 0.022301698103547096,
      "step": 156
    },
    {
      "epoch": 10.16,
      "grad_norm": 6.74313497543335,
      "learning_rate": 4.969661229192477e-07,
      "logits/chosen": -2.7252540588378906,
      "logits/rejected": -2.709249258041382,
      "logps/chosen": -126.21170043945312,
      "logps/rejected": -105.25879669189453,
      "loss": 0.4607,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.6133719682693481,
      "rewards/margins": 0.6076129674911499,
      "rewards/rejected": 0.005758981220424175,
      "step": 160
    },
    {
      "epoch": 10.16,
      "eval_logits/chosen": -2.807128429412842,
      "eval_logits/rejected": -2.8889076709747314,
      "eval_logps/chosen": -128.48606872558594,
      "eval_logps/rejected": -97.06517028808594,
      "eval_loss": 0.5175719857215881,
      "eval_rewards/accuracies": 0.8214285969734192,
      "eval_rewards/chosen": 0.5583271384239197,
      "eval_rewards/margins": 0.4539553225040436,
      "eval_rewards/rejected": 0.10437185317277908,
      "eval_runtime": 19.3848,
      "eval_samples_per_second": 1.444,
      "eval_steps_per_second": 1.444,
      "step": 160
    },
    {
      "epoch": 10.41,
      "grad_norm": 5.443469524383545,
      "learning_rate": 4.967880800636598e-07,
      "logits/chosen": -2.7496747970581055,
      "logits/rejected": -2.768813133239746,
      "logps/chosen": -122.11544799804688,
      "logps/rejected": -94.54540252685547,
      "loss": 0.436,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.6616504788398743,
      "rewards/margins": 0.67127525806427,
      "rewards/rejected": -0.009624744765460491,
      "step": 164
    },
    {
      "epoch": 10.67,
      "grad_norm": 5.551913738250732,
      "learning_rate": 4.966049946190263e-07,
      "logits/chosen": -2.7052814960479736,
      "logits/rejected": -2.7216036319732666,
      "logps/chosen": -131.8173065185547,
      "logps/rejected": -102.79641723632812,
      "loss": 0.4326,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.7013098001480103,
      "rewards/margins": 0.6718720197677612,
      "rewards/rejected": 0.02943783439695835,
      "step": 168
    },
    {
      "epoch": 10.92,
      "grad_norm": 5.826549530029297,
      "learning_rate": 4.964168703263086e-07,
      "logits/chosen": -2.701007604598999,
      "logits/rejected": -2.741464614868164,
      "logps/chosen": -116.08711242675781,
      "logps/rejected": -94.73246765136719,
      "loss": 0.446,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.6390998363494873,
      "rewards/margins": 0.6307864785194397,
      "rewards/rejected": 0.008313396014273167,
      "step": 172
    },
    {
      "epoch": 11.17,
      "grad_norm": 6.066051483154297,
      "learning_rate": 4.962237110294259e-07,
      "logits/chosen": -2.7041149139404297,
      "logits/rejected": -2.731684684753418,
      "logps/chosen": -124.84115600585938,
      "logps/rejected": -103.8166732788086,
      "loss": 0.4294,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.6835889220237732,
      "rewards/margins": 0.6921644806861877,
      "rewards/rejected": -0.00857558287680149,
      "step": 176
    },
    {
      "epoch": 11.43,
      "grad_norm": 5.8145012855529785,
      "learning_rate": 4.960255206751773e-07,
      "logits/chosen": -2.7352046966552734,
      "logits/rejected": -2.7856600284576416,
      "logps/chosen": -123.47644805908203,
      "logps/rejected": -93.96688079833984,
      "loss": 0.4056,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.7594637274742126,
      "rewards/margins": 0.7790957093238831,
      "rewards/rejected": -0.019631966948509216,
      "step": 180
    },
    {
      "epoch": 11.68,
      "grad_norm": 5.962335109710693,
      "learning_rate": 4.958223033131609e-07,
      "logits/chosen": -2.740914821624756,
      "logits/rejected": -2.7585391998291016,
      "logps/chosen": -124.35005950927734,
      "logps/rejected": -105.05398559570312,
      "loss": 0.4293,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.6812430620193481,
      "rewards/margins": 0.6962765455245972,
      "rewards/rejected": -0.015033497475087643,
      "step": 184
    },
    {
      "epoch": 11.94,
      "grad_norm": 5.529379367828369,
      "learning_rate": 4.956140630956907e-07,
      "logits/chosen": -2.7024734020233154,
      "logits/rejected": -2.6746203899383545,
      "logps/chosen": -125.29593658447266,
      "logps/rejected": -94.42231750488281,
      "loss": 0.3963,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.7892162799835205,
      "rewards/margins": 0.8050926327705383,
      "rewards/rejected": -0.01587628945708275,
      "step": 188
    },
    {
      "epoch": 12.19,
      "grad_norm": 5.774294376373291,
      "learning_rate": 4.954008042777125e-07,
      "logits/chosen": -2.7662813663482666,
      "logits/rejected": -2.8167452812194824,
      "logps/chosen": -124.10614776611328,
      "logps/rejected": -94.9889907836914,
      "loss": 0.3728,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.8182777166366577,
      "rewards/margins": 0.8937400579452515,
      "rewards/rejected": -0.07546230405569077,
      "step": 192
    },
    {
      "epoch": 12.44,
      "grad_norm": 4.990906715393066,
      "learning_rate": 4.951825312167159e-07,
      "logits/chosen": -2.7352771759033203,
      "logits/rejected": -2.7206149101257324,
      "logps/chosen": -116.98539733886719,
      "logps/rejected": -96.68843078613281,
      "loss": 0.4435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.7208267450332642,
      "rewards/margins": 0.6662319302558899,
      "rewards/rejected": 0.05459484085440636,
      "step": 196
    },
    {
      "epoch": 12.7,
      "grad_norm": 7.139024257659912,
      "learning_rate": 4.949592483726464e-07,
      "logits/chosen": -2.728516101837158,
      "logits/rejected": -2.732177257537842,
      "logps/chosen": -125.51838684082031,
      "logps/rejected": -103.7790298461914,
      "loss": 0.391,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.7955451011657715,
      "rewards/margins": 0.8273122310638428,
      "rewards/rejected": -0.03176717832684517,
      "step": 200
    },
    {
      "epoch": 12.7,
      "eval_logits/chosen": -2.8053383827209473,
      "eval_logits/rejected": -2.8861937522888184,
      "eval_logps/chosen": -127.08057403564453,
      "eval_logps/rejected": -96.8802261352539,
      "eval_loss": 0.4858916103839874,
      "eval_rewards/accuracies": 0.8214285969734192,
      "eval_rewards/chosen": 0.6988765597343445,
      "eval_rewards/margins": 0.5760089755058289,
      "eval_rewards/rejected": 0.12286762148141861,
      "eval_runtime": 19.4173,
      "eval_samples_per_second": 1.442,
      "eval_steps_per_second": 1.442,
      "step": 200
    },
    {
      "epoch": 12.95,
      "grad_norm": 5.450782299041748,
      "learning_rate": 4.947309603078138e-07,
      "logits/chosen": -2.6719353199005127,
      "logits/rejected": -2.7134647369384766,
      "logps/chosen": -124.22660827636719,
      "logps/rejected": -99.73461151123047,
      "loss": 0.4062,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.7935754060745239,
      "rewards/margins": 0.783437192440033,
      "rewards/rejected": 0.010138224810361862,
      "step": 204
    },
    {
      "epoch": 13.21,
      "grad_norm": 5.752992153167725,
      "learning_rate": 4.944976716867984e-07,
      "logits/chosen": -2.7026755809783936,
      "logits/rejected": -2.698920726776123,
      "logps/chosen": -123.63475036621094,
      "logps/rejected": -107.06332397460938,
      "loss": 0.3952,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.8116875886917114,
      "rewards/margins": 0.8335397839546204,
      "rewards/rejected": -0.02185220457613468,
      "step": 208
    },
    {
      "epoch": 13.46,
      "grad_norm": 5.456747531890869,
      "learning_rate": 4.942593872763566e-07,
      "logits/chosen": -2.701968193054199,
      "logits/rejected": -2.786785840988159,
      "logps/chosen": -124.91243743896484,
      "logps/rejected": -101.95451354980469,
      "loss": 0.3673,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.8831655979156494,
      "rewards/margins": 0.9259040951728821,
      "rewards/rejected": -0.04273851588368416,
      "step": 212
    },
    {
      "epoch": 13.71,
      "grad_norm": 4.753533363342285,
      "learning_rate": 4.940161119453231e-07,
      "logits/chosen": -2.7109193801879883,
      "logits/rejected": -2.6961448192596436,
      "logps/chosen": -123.21333312988281,
      "logps/rejected": -92.74653625488281,
      "loss": 0.3832,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.8407853245735168,
      "rewards/margins": 0.8698095083236694,
      "rewards/rejected": -0.02902432158589363,
      "step": 216
    },
    {
      "epoch": 13.97,
      "grad_norm": 5.374657154083252,
      "learning_rate": 4.937678506645115e-07,
      "logits/chosen": -2.7288832664489746,
      "logits/rejected": -2.7349352836608887,
      "logps/chosen": -123.1498031616211,
      "logps/rejected": -95.77909851074219,
      "loss": 0.3644,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.8925550580024719,
      "rewards/margins": 0.9183064699172974,
      "rewards/rejected": -0.025751333683729172,
      "step": 220
    },
    {
      "epoch": 14.22,
      "grad_norm": 5.842701435089111,
      "learning_rate": 4.935146085066125e-07,
      "logits/chosen": -2.7030577659606934,
      "logits/rejected": -2.7200875282287598,
      "logps/chosen": -120.81951904296875,
      "logps/rejected": -100.53494262695312,
      "loss": 0.3581,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.9156714677810669,
      "rewards/margins": 0.9704564809799194,
      "rewards/rejected": -0.05478499457240105,
      "step": 224
    },
    {
      "epoch": 14.48,
      "grad_norm": 5.020945072174072,
      "learning_rate": 4.932563906460904e-07,
      "logits/chosen": -2.722069501876831,
      "logits/rejected": -2.731358051300049,
      "logps/chosen": -123.65387725830078,
      "logps/rejected": -103.83390808105469,
      "loss": 0.391,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.8787897229194641,
      "rewards/margins": 0.8743190765380859,
      "rewards/rejected": 0.004470685496926308,
      "step": 228
    },
    {
      "epoch": 14.73,
      "grad_norm": 5.041106700897217,
      "learning_rate": 4.929932023590776e-07,
      "logits/chosen": -2.735006093978882,
      "logits/rejected": -2.7546191215515137,
      "logps/chosen": -124.65998840332031,
      "logps/rejected": -101.49640655517578,
      "loss": 0.3586,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.9206160306930542,
      "rewards/margins": 0.9452594518661499,
      "rewards/rejected": -0.024643439799547195,
      "step": 232
    },
    {
      "epoch": 14.98,
      "grad_norm": 5.318466663360596,
      "learning_rate": 4.927250490232664e-07,
      "logits/chosen": -2.6883015632629395,
      "logits/rejected": -2.718212842941284,
      "logps/chosen": -120.88870239257812,
      "logps/rejected": -94.79769134521484,
      "loss": 0.3564,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.9339860677719116,
      "rewards/margins": 0.9901734590530396,
      "rewards/rejected": -0.05618742108345032,
      "step": 236
    },
    {
      "epoch": 15.24,
      "grad_norm": 5.438621997833252,
      "learning_rate": 4.924519361177992e-07,
      "logits/chosen": -2.7316794395446777,
      "logits/rejected": -2.812800407409668,
      "logps/chosen": -120.06542205810547,
      "logps/rejected": -97.61270141601562,
      "loss": 0.3471,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.9851706624031067,
      "rewards/margins": 1.039989948272705,
      "rewards/rejected": -0.05481935292482376,
      "step": 240
    },
    {
      "epoch": 15.24,
      "eval_logits/chosen": -2.8022568225860596,
      "eval_logits/rejected": -2.882326602935791,
      "eval_logps/chosen": -125.87542724609375,
      "eval_logps/rejected": -96.83832550048828,
      "eval_loss": 0.46029001474380493,
      "eval_rewards/accuracies": 0.7857142686843872,
      "eval_rewards/chosen": 0.8193913102149963,
      "eval_rewards/margins": 0.6923348307609558,
      "eval_rewards/rejected": 0.12705650925636292,
      "eval_runtime": 19.4065,
      "eval_samples_per_second": 1.443,
      "eval_steps_per_second": 1.443,
      "step": 240
    },
    {
      "epoch": 15.49,
      "grad_norm": 5.301855087280273,
      "learning_rate": 4.921738692231572e-07,
      "logits/chosen": -2.661013126373291,
      "logits/rejected": -2.666978359222412,
      "logps/chosen": -121.69336700439453,
      "logps/rejected": -101.86541748046875,
      "loss": 0.3574,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.9682250022888184,
      "rewards/margins": 0.9734476208686829,
      "rewards/rejected": -0.0052225226536393166,
      "step": 244
    },
    {
      "epoch": 15.75,
      "grad_norm": 4.478713512420654,
      "learning_rate": 4.918908540210451e-07,
      "logits/chosen": -2.702775239944458,
      "logits/rejected": -2.7098357677459717,
      "logps/chosen": -119.93941497802734,
      "logps/rejected": -94.76625061035156,
      "loss": 0.3517,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.8942417502403259,
      "rewards/margins": 0.9958410263061523,
      "rewards/rejected": -0.10159924626350403,
      "step": 248
    },
    {
      "epoch": 16.0,
      "grad_norm": 4.398434162139893,
      "learning_rate": 4.916028962942762e-07,
      "logits/chosen": -2.7399990558624268,
      "logits/rejected": -2.7476425170898438,
      "logps/chosen": -124.2032470703125,
      "logps/rejected": -101.86943054199219,
      "loss": 0.3372,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.0369476079940796,
      "rewards/margins": 1.0664609670639038,
      "rewards/rejected": -0.029513416811823845,
      "step": 252
    },
    {
      "epoch": 16.25,
      "grad_norm": 4.430797576904297,
      "learning_rate": 4.913100019266536e-07,
      "logits/chosen": -2.6950950622558594,
      "logits/rejected": -2.7181689739227295,
      "logps/chosen": -123.97085571289062,
      "logps/rejected": -97.85747528076172,
      "loss": 0.3226,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.0282835960388184,
      "rewards/margins": 1.1182292699813843,
      "rewards/rejected": -0.08994561433792114,
      "step": 256
    },
    {
      "epoch": 16.51,
      "grad_norm": 6.383476734161377,
      "learning_rate": 4.910121769028503e-07,
      "logits/chosen": -2.7224864959716797,
      "logits/rejected": -2.727613925933838,
      "logps/chosen": -117.91517639160156,
      "logps/rejected": -99.52539825439453,
      "loss": 0.3658,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.9342272877693176,
      "rewards/margins": 0.9777376651763916,
      "rewards/rejected": -0.04351034015417099,
      "step": 260
    },
    {
      "epoch": 16.76,
      "grad_norm": 4.453210353851318,
      "learning_rate": 4.907094273082864e-07,
      "logits/chosen": -2.7149100303649902,
      "logits/rejected": -2.710146188735962,
      "logps/chosen": -126.07495880126953,
      "logps/rejected": -100.1856689453125,
      "loss": 0.3162,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.1231064796447754,
      "rewards/margins": 1.175417423248291,
      "rewards/rejected": -0.05231098458170891,
      "step": 264
    },
    {
      "epoch": 17.02,
      "grad_norm": 4.718712329864502,
      "learning_rate": 4.904017593290055e-07,
      "logits/chosen": -2.710063934326172,
      "logits/rejected": -2.7535319328308105,
      "logps/chosen": -117.01451873779297,
      "logps/rejected": -99.95758819580078,
      "loss": 0.322,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.033459186553955,
      "rewards/margins": 1.1137871742248535,
      "rewards/rejected": -0.08032786101102829,
      "step": 268
    },
    {
      "epoch": 17.27,
      "grad_norm": 5.164140701293945,
      "learning_rate": 4.900891792515478e-07,
      "logits/chosen": -2.7342586517333984,
      "logits/rejected": -2.7561776638031006,
      "logps/chosen": -118.80335998535156,
      "logps/rejected": -104.33666229248047,
      "loss": 0.3314,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.078546404838562,
      "rewards/margins": 1.0902469158172607,
      "rewards/rejected": -0.011700518429279327,
      "step": 272
    },
    {
      "epoch": 17.52,
      "grad_norm": 4.392764568328857,
      "learning_rate": 4.897716934628217e-07,
      "logits/chosen": -2.6796340942382812,
      "logits/rejected": -2.6666793823242188,
      "logps/chosen": -122.01671600341797,
      "logps/rejected": -97.74385070800781,
      "loss": 0.3047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0594760179519653,
      "rewards/margins": 1.1904582977294922,
      "rewards/rejected": -0.1309822052717209,
      "step": 276
    },
    {
      "epoch": 17.78,
      "grad_norm": 4.647542953491211,
      "learning_rate": 4.894493084499736e-07,
      "logits/chosen": -2.7203075885772705,
      "logits/rejected": -2.7299890518188477,
      "logps/chosen": -120.08210754394531,
      "logps/rejected": -98.8072738647461,
      "loss": 0.3496,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 1.0658477544784546,
      "rewards/margins": 1.065575122833252,
      "rewards/rejected": 0.0002726661041378975,
      "step": 280
    },
    {
      "epoch": 17.78,
      "eval_logits/chosen": -2.798058271408081,
      "eval_logits/rejected": -2.8771989345550537,
      "eval_logps/chosen": -124.81001281738281,
      "eval_logps/rejected": -96.83968353271484,
      "eval_loss": 0.44096532464027405,
      "eval_rewards/accuracies": 0.7857142686843872,
      "eval_rewards/chosen": 0.9259332418441772,
      "eval_rewards/margins": 0.799013078212738,
      "eval_rewards/rejected": 0.12692023813724518,
      "eval_runtime": 19.3925,
      "eval_samples_per_second": 1.444,
      "eval_steps_per_second": 1.444,
      "step": 280
    },
    {
      "epoch": 18.03,
      "grad_norm": 4.158952236175537,
      "learning_rate": 4.891220308002547e-07,
      "logits/chosen": -2.6795365810394287,
      "logits/rejected": -2.7270519733428955,
      "logps/chosen": -120.39765167236328,
      "logps/rejected": -100.02141571044922,
      "loss": 0.319,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.0584629774093628,
      "rewards/margins": 1.19419264793396,
      "rewards/rejected": -0.13572952151298523,
      "step": 284
    },
    {
      "epoch": 18.29,
      "grad_norm": 4.605132579803467,
      "learning_rate": 4.887898672008871e-07,
      "logits/chosen": -2.696953296661377,
      "logits/rejected": -2.718316078186035,
      "logps/chosen": -120.32958984375,
      "logps/rejected": -101.76454162597656,
      "loss": 0.331,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 1.0935906171798706,
      "rewards/margins": 1.1072776317596436,
      "rewards/rejected": -0.013687020167708397,
      "step": 288
    },
    {
      "epoch": 18.54,
      "grad_norm": 5.824638843536377,
      "learning_rate": 4.884528244389268e-07,
      "logits/chosen": -2.6889617443084717,
      "logits/rejected": -2.690692663192749,
      "logps/chosen": -120.37040710449219,
      "logps/rejected": -98.27632141113281,
      "loss": 0.3204,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 1.1049582958221436,
      "rewards/margins": 1.175376057624817,
      "rewards/rejected": -0.07041767239570618,
      "step": 292
    },
    {
      "epoch": 18.79,
      "grad_norm": 5.431331634521484,
      "learning_rate": 4.881109094011253e-07,
      "logits/chosen": -2.740906238555908,
      "logits/rejected": -2.725914478302002,
      "logps/chosen": -124.35327911376953,
      "logps/rejected": -100.71537780761719,
      "loss": 0.2884,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2057325839996338,
      "rewards/margins": 1.3113868236541748,
      "rewards/rejected": -0.10565409809350967,
      "step": 296
    },
    {
      "epoch": 19.05,
      "grad_norm": 4.33558988571167,
      "learning_rate": 4.877641290737883e-07,
      "logits/chosen": -2.700420379638672,
      "logits/rejected": -2.7527971267700195,
      "logps/chosen": -117.01264953613281,
      "logps/rejected": -101.50834655761719,
      "loss": 0.2757,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.150115728378296,
      "rewards/margins": 1.34402596950531,
      "rewards/rejected": -0.19391010701656342,
      "step": 300
    },
    {
      "epoch": 19.3,
      "grad_norm": 4.987298488616943,
      "learning_rate": 4.874124905426338e-07,
      "logits/chosen": -2.715477705001831,
      "logits/rejected": -2.760146141052246,
      "logps/chosen": -117.20336151123047,
      "logps/rejected": -94.89369201660156,
      "loss": 0.3244,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 1.107708215713501,
      "rewards/margins": 1.1569105386734009,
      "rewards/rejected": -0.04920227825641632,
      "step": 304
    },
    {
      "epoch": 19.56,
      "grad_norm": 4.090747833251953,
      "learning_rate": 4.870560009926464e-07,
      "logits/chosen": -2.6785225868225098,
      "logits/rejected": -2.6984167098999023,
      "logps/chosen": -112.27914428710938,
      "logps/rejected": -94.59768676757812,
      "loss": 0.2833,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.2028326988220215,
      "rewards/margins": 1.334948182106018,
      "rewards/rejected": -0.13211557269096375,
      "step": 308
    },
    {
      "epoch": 19.81,
      "grad_norm": 5.0767083168029785,
      "learning_rate": 4.866946677079314e-07,
      "logits/chosen": -2.712416410446167,
      "logits/rejected": -2.7296035289764404,
      "logps/chosen": -121.65861511230469,
      "logps/rejected": -106.11611938476562,
      "loss": 0.2893,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.1842504739761353,
      "rewards/margins": 1.3191744089126587,
      "rewards/rejected": -0.13492396473884583,
      "step": 312
    },
    {
      "epoch": 20.06,
      "grad_norm": 4.1403326988220215,
      "learning_rate": 4.863284980715648e-07,
      "logits/chosen": -2.658827781677246,
      "logits/rejected": -2.631199836730957,
      "logps/chosen": -129.3588409423828,
      "logps/rejected": -103.25465393066406,
      "loss": 0.2678,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.291969895362854,
      "rewards/margins": 1.4036282300949097,
      "rewards/rejected": -0.11165820807218552,
      "step": 316
    },
    {
      "epoch": 20.32,
      "grad_norm": 4.350497245788574,
      "learning_rate": 4.859574995654441e-07,
      "logits/chosen": -2.761063814163208,
      "logits/rejected": -2.7751922607421875,
      "logps/chosen": -114.76087951660156,
      "logps/rejected": -100.04124450683594,
      "loss": 0.298,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.1642746925354004,
      "rewards/margins": 1.3020979166030884,
      "rewards/rejected": -0.13782335817813873,
      "step": 320
    },
    {
      "epoch": 20.32,
      "eval_logits/chosen": -2.792503595352173,
      "eval_logits/rejected": -2.870570421218872,
      "eval_logps/chosen": -123.9360122680664,
      "eval_logps/rejected": -96.95860290527344,
      "eval_loss": 0.4256473481655121,
      "eval_rewards/accuracies": 0.8214285969734192,
      "eval_rewards/chosen": 1.0133329629898071,
      "eval_rewards/margins": 0.8983038663864136,
      "eval_rewards/rejected": 0.11502902954816818,
      "eval_runtime": 19.3701,
      "eval_samples_per_second": 1.446,
      "eval_steps_per_second": 1.446,
      "step": 320
    },
    {
      "epoch": 20.57,
      "grad_norm": 4.334249973297119,
      "learning_rate": 4.855816797701336e-07,
      "logits/chosen": -2.690117359161377,
      "logits/rejected": -2.7377476692199707,
      "logps/chosen": -116.01565551757812,
      "logps/rejected": -102.1231689453125,
      "loss": 0.2872,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 1.2394335269927979,
      "rewards/margins": 1.3586478233337402,
      "rewards/rejected": -0.11921422928571701,
      "step": 324
    },
    {
      "epoch": 20.83,
      "grad_norm": 4.597221851348877,
      "learning_rate": 4.85201046364711e-07,
      "logits/chosen": -2.741600751876831,
      "logits/rejected": -2.712954044342041,
      "logps/chosen": -124.32752990722656,
      "logps/rejected": -101.2073974609375,
      "loss": 0.2676,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.22231924533844,
      "rewards/margins": 1.4293614625930786,
      "rewards/rejected": -0.20704227685928345,
      "step": 328
    },
    {
      "epoch": 21.08,
      "grad_norm": 3.682258367538452,
      "learning_rate": 4.848156071266095e-07,
      "logits/chosen": -2.6585566997528076,
      "logits/rejected": -2.704587459564209,
      "logps/chosen": -120.91087341308594,
      "logps/rejected": -98.83493041992188,
      "loss": 0.2705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2277172803878784,
      "rewards/margins": 1.3790152072906494,
      "rewards/rejected": -0.15129785239696503,
      "step": 332
    },
    {
      "epoch": 21.33,
      "grad_norm": 3.7775039672851562,
      "learning_rate": 4.844253699314596e-07,
      "logits/chosen": -2.7104837894439697,
      "logits/rejected": -2.7378451824188232,
      "logps/chosen": -117.22598266601562,
      "logps/rejected": -97.59563446044922,
      "loss": 0.2755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2548604011535645,
      "rewards/margins": 1.3694878816604614,
      "rewards/rejected": -0.11462761461734772,
      "step": 336
    },
    {
      "epoch": 21.59,
      "grad_norm": 4.84611701965332,
      "learning_rate": 4.840303427529273e-07,
      "logits/chosen": -2.6837968826293945,
      "logits/rejected": -2.7071595191955566,
      "logps/chosen": -121.88581085205078,
      "logps/rejected": -102.45454406738281,
      "loss": 0.286,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.2159500122070312,
      "rewards/margins": 1.3315218687057495,
      "rewards/rejected": -0.11557181179523468,
      "step": 340
    },
    {
      "epoch": 21.84,
      "grad_norm": 3.760068655014038,
      "learning_rate": 4.836305336625523e-07,
      "logits/chosen": -2.709066390991211,
      "logits/rejected": -2.73175048828125,
      "logps/chosen": -117.1901626586914,
      "logps/rejected": -105.62377166748047,
      "loss": 0.2661,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.3225661516189575,
      "rewards/margins": 1.445369005203247,
      "rewards/rejected": -0.12280277907848358,
      "step": 344
    },
    {
      "epoch": 22.1,
      "grad_norm": 3.8488616943359375,
      "learning_rate": 4.832259508295822e-07,
      "logits/chosen": -2.7598538398742676,
      "logits/rejected": -2.776313066482544,
      "logps/chosen": -117.15644836425781,
      "logps/rejected": -93.95286560058594,
      "loss": 0.2711,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 1.3003535270690918,
      "rewards/margins": 1.5021978616714478,
      "rewards/rejected": -0.20184442400932312,
      "step": 348
    },
    {
      "epoch": 22.35,
      "grad_norm": 3.152104377746582,
      "learning_rate": 4.828166025208058e-07,
      "logits/chosen": -2.6503114700317383,
      "logits/rejected": -2.6512064933776855,
      "logps/chosen": -118.29179382324219,
      "logps/rejected": -97.83879852294922,
      "loss": 0.2394,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.2750773429870605,
      "rewards/margins": 1.585729718208313,
      "rewards/rejected": -0.31065240502357483,
      "step": 352
    },
    {
      "epoch": 22.6,
      "grad_norm": 4.1737165451049805,
      "learning_rate": 4.824024971003845e-07,
      "logits/chosen": -2.708979845046997,
      "logits/rejected": -2.718264579772949,
      "logps/chosen": -113.75922393798828,
      "logps/rejected": -100.5313720703125,
      "loss": 0.2626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3227206468582153,
      "rewards/margins": 1.4637939929962158,
      "rewards/rejected": -0.1410733461380005,
      "step": 356
    },
    {
      "epoch": 22.86,
      "grad_norm": 4.2186737060546875,
      "learning_rate": 4.819836430296808e-07,
      "logits/chosen": -2.688643217086792,
      "logits/rejected": -2.7126376628875732,
      "logps/chosen": -119.3924560546875,
      "logps/rejected": -102.88687896728516,
      "loss": 0.272,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.3319997787475586,
      "rewards/margins": 1.3775808811187744,
      "rewards/rejected": -0.04558102786540985,
      "step": 360
    },
    {
      "epoch": 22.86,
      "eval_logits/chosen": -2.7863781452178955,
      "eval_logits/rejected": -2.8632652759552,
      "eval_logps/chosen": -123.22322082519531,
      "eval_logps/rejected": -97.17105865478516,
      "eval_loss": 0.4147447645664215,
      "eval_rewards/accuracies": 0.8214285969734192,
      "eval_rewards/chosen": 1.0846121311187744,
      "eval_rewards/margins": 0.9908290505409241,
      "eval_rewards/rejected": 0.09378316253423691,
      "eval_runtime": 19.3953,
      "eval_samples_per_second": 1.444,
      "eval_steps_per_second": 1.444,
      "step": 360
    },
    {
      "epoch": 23.11,
      "grad_norm": 3.8801398277282715,
      "learning_rate": 4.815600488670862e-07,
      "logits/chosen": -2.661937713623047,
      "logits/rejected": -2.69406795501709,
      "logps/chosen": -120.42941284179688,
      "logps/rejected": -107.56249237060547,
      "loss": 0.2463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.303568959236145,
      "rewards/margins": 1.5796161890029907,
      "rewards/rejected": -0.2760472595691681,
      "step": 364
    },
    {
      "epoch": 23.37,
      "grad_norm": 3.6953530311584473,
      "learning_rate": 4.811317232678456e-07,
      "logits/chosen": -2.696389675140381,
      "logits/rejected": -2.7162017822265625,
      "logps/chosen": -115.95695495605469,
      "logps/rejected": -95.26593017578125,
      "loss": 0.2557,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.3928656578063965,
      "rewards/margins": 1.5144150257110596,
      "rewards/rejected": -0.12154923379421234,
      "step": 368
    },
    {
      "epoch": 23.62,
      "grad_norm": 4.2311553955078125,
      "learning_rate": 4.806986749838806e-07,
      "logits/chosen": -2.735870599746704,
      "logits/rejected": -2.7359132766723633,
      "logps/chosen": -120.84663391113281,
      "logps/rejected": -107.17831420898438,
      "loss": 0.2922,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.2136850357055664,
      "rewards/margins": 1.3251327276229858,
      "rewards/rejected": -0.11144763976335526,
      "step": 372
    },
    {
      "epoch": 23.87,
      "grad_norm": 3.250326633453369,
      "learning_rate": 4.802609128636112e-07,
      "logits/chosen": -2.6809797286987305,
      "logits/rejected": -2.700901508331299,
      "logps/chosen": -121.92070007324219,
      "logps/rejected": -99.91931915283203,
      "loss": 0.2062,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.4374094009399414,
      "rewards/margins": 1.8276853561401367,
      "rewards/rejected": -0.3902759850025177,
      "step": 376
    },
    {
      "epoch": 24.13,
      "grad_norm": 4.59958553314209,
      "learning_rate": 4.798184458517744e-07,
      "logits/chosen": -2.7332396507263184,
      "logits/rejected": -2.768554210662842,
      "logps/chosen": -114.1657943725586,
      "logps/rejected": -100.7848892211914,
      "loss": 0.2353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4016469717025757,
      "rewards/margins": 1.5988764762878418,
      "rewards/rejected": -0.19722968339920044,
      "step": 380
    },
    {
      "epoch": 24.38,
      "grad_norm": 3.6598093509674072,
      "learning_rate": 4.793712829892414e-07,
      "logits/chosen": -2.7024924755096436,
      "logits/rejected": -2.7085440158843994,
      "logps/chosen": -114.50930786132812,
      "logps/rejected": -104.57618713378906,
      "loss": 0.2538,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4238942861557007,
      "rewards/margins": 1.5259473323822021,
      "rewards/rejected": -0.10205309092998505,
      "step": 384
    },
    {
      "epoch": 24.63,
      "grad_norm": 3.3857998847961426,
      "learning_rate": 4.789194334128338e-07,
      "logits/chosen": -2.7139151096343994,
      "logits/rejected": -2.7087197303771973,
      "logps/chosen": -121.57815551757812,
      "logps/rejected": -102.66044616699219,
      "loss": 0.2324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3619202375411987,
      "rewards/margins": 1.667473316192627,
      "rewards/rejected": -0.305553138256073,
      "step": 388
    },
    {
      "epoch": 24.89,
      "grad_norm": 4.038039207458496,
      "learning_rate": 4.784629063551354e-07,
      "logits/chosen": -2.651282548904419,
      "logits/rejected": -2.679689884185791,
      "logps/chosen": -118.9875717163086,
      "logps/rejected": -97.62678527832031,
      "loss": 0.2304,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.3316906690597534,
      "rewards/margins": 1.7062404155731201,
      "rewards/rejected": -0.3745496869087219,
      "step": 392
    },
    {
      "epoch": 25.14,
      "grad_norm": 3.0911149978637695,
      "learning_rate": 4.780017111443047e-07,
      "logits/chosen": -2.718074083328247,
      "logits/rejected": -2.7447762489318848,
      "logps/chosen": -121.20558166503906,
      "logps/rejected": -105.63926696777344,
      "loss": 0.2411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4981815814971924,
      "rewards/margins": 1.644660472869873,
      "rewards/rejected": -0.14647886157035828,
      "step": 396
    },
    {
      "epoch": 25.4,
      "grad_norm": 3.236288070678711,
      "learning_rate": 4.775358572038845e-07,
      "logits/chosen": -2.693341016769409,
      "logits/rejected": -2.681511878967285,
      "logps/chosen": -114.28794860839844,
      "logps/rejected": -91.69850158691406,
      "loss": 0.259,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.3284053802490234,
      "rewards/margins": 1.5037521123886108,
      "rewards/rejected": -0.17534679174423218,
      "step": 400
    },
    {
      "epoch": 25.4,
      "eval_logits/chosen": -2.7796428203582764,
      "eval_logits/rejected": -2.8552372455596924,
      "eval_logps/chosen": -122.60997009277344,
      "eval_logps/rejected": -97.46685028076172,
      "eval_loss": 0.4054030478000641,
      "eval_rewards/accuracies": 0.7857142686843872,
      "eval_rewards/chosen": 1.1459368467330933,
      "eval_rewards/margins": 1.0817322731018066,
      "eval_rewards/rejected": 0.06420455127954483,
      "eval_runtime": 19.3983,
      "eval_samples_per_second": 1.443,
      "eval_steps_per_second": 1.443,
      "step": 400
    },
    {
      "epoch": 25.65,
      "grad_norm": 3.2894155979156494,
      "learning_rate": 4.770653540526078e-07,
      "logits/chosen": -2.652012825012207,
      "logits/rejected": -2.690704345703125,
      "logps/chosen": -118.52864837646484,
      "logps/rejected": -102.0433578491211,
      "loss": 0.1918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5184407234191895,
      "rewards/margins": 1.8730953931808472,
      "rewards/rejected": -0.3546547293663025,
      "step": 404
    },
    {
      "epoch": 25.9,
      "grad_norm": 4.958095550537109,
      "learning_rate": 4.765902113042053e-07,
      "logits/chosen": -2.7191715240478516,
      "logits/rejected": -2.701702833175659,
      "logps/chosen": -115.93360900878906,
      "logps/rejected": -100.49573516845703,
      "loss": 0.2536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.357507348060608,
      "rewards/margins": 1.542129635810852,
      "rewards/rejected": -0.18462230265140533,
      "step": 408
    },
    {
      "epoch": 26.16,
      "grad_norm": 3.1129214763641357,
      "learning_rate": 4.7611043866720737e-07,
      "logits/chosen": -2.6934893131256104,
      "logits/rejected": -2.7433109283447266,
      "logps/chosen": -117.78741455078125,
      "logps/rejected": -102.46746063232422,
      "loss": 0.2137,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.484704613685608,
      "rewards/margins": 1.8401193618774414,
      "rewards/rejected": -0.3554147481918335,
      "step": 412
    },
    {
      "epoch": 26.41,
      "grad_norm": 3.2823047637939453,
      "learning_rate": 4.7562604594474643e-07,
      "logits/chosen": -2.6928577423095703,
      "logits/rejected": -2.708937644958496,
      "logps/chosen": -124.94667053222656,
      "logps/rejected": -106.61329650878906,
      "loss": 0.2137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.471519112586975,
      "rewards/margins": 1.7852158546447754,
      "rewards/rejected": -0.31369656324386597,
      "step": 416
    },
    {
      "epoch": 26.67,
      "grad_norm": 3.7071120738983154,
      "learning_rate": 4.7513704303435674e-07,
      "logits/chosen": -2.670248031616211,
      "logits/rejected": -2.6764914989471436,
      "logps/chosen": -110.71532440185547,
      "logps/rejected": -101.84510803222656,
      "loss": 0.229,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.4525586366653442,
      "rewards/margins": 1.6690635681152344,
      "rewards/rejected": -0.21650481224060059,
      "step": 420
    },
    {
      "epoch": 26.92,
      "grad_norm": 3.449319362640381,
      "learning_rate": 4.7464343992777165e-07,
      "logits/chosen": -2.688829183578491,
      "logits/rejected": -2.7286489009857178,
      "logps/chosen": -115.69140625,
      "logps/rejected": -103.97221374511719,
      "loss": 0.2231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4910831451416016,
      "rewards/margins": 1.7472798824310303,
      "rewards/rejected": -0.2561967670917511,
      "step": 424
    },
    {
      "epoch": 27.17,
      "grad_norm": 2.4958574771881104,
      "learning_rate": 4.741452467107199e-07,
      "logits/chosen": -2.6915664672851562,
      "logits/rejected": -2.7021422386169434,
      "logps/chosen": -110.91499328613281,
      "logps/rejected": -98.70269775390625,
      "loss": 0.233,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.4354866743087769,
      "rewards/margins": 1.7168078422546387,
      "rewards/rejected": -0.28132107853889465,
      "step": 428
    },
    {
      "epoch": 27.43,
      "grad_norm": 3.7154319286346436,
      "learning_rate": 4.7364247356271927e-07,
      "logits/chosen": -2.6695199012756348,
      "logits/rejected": -2.6698789596557617,
      "logps/chosen": -118.46913146972656,
      "logps/rejected": -102.86263275146484,
      "loss": 0.2337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4410640001296997,
      "rewards/margins": 1.6770535707473755,
      "rewards/rejected": -0.23598948121070862,
      "step": 432
    },
    {
      "epoch": 27.68,
      "grad_norm": 3.101527690887451,
      "learning_rate": 4.731351307568687e-07,
      "logits/chosen": -2.6910083293914795,
      "logits/rejected": -2.6801414489746094,
      "logps/chosen": -115.44275665283203,
      "logps/rejected": -97.7877197265625,
      "loss": 0.2,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5193870067596436,
      "rewards/margins": 1.8750090599060059,
      "rewards/rejected": -0.35562214255332947,
      "step": 436
    },
    {
      "epoch": 27.94,
      "grad_norm": 3.9017672538757324,
      "learning_rate": 4.7262322865963846e-07,
      "logits/chosen": -2.7122743129730225,
      "logits/rejected": -2.723294258117676,
      "logps/chosen": -116.90724182128906,
      "logps/rejected": -104.45236206054688,
      "loss": 0.1959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5187357664108276,
      "rewards/margins": 1.918578863143921,
      "rewards/rejected": -0.39984312653541565,
      "step": 440
    },
    {
      "epoch": 27.94,
      "eval_logits/chosen": -2.7724335193634033,
      "eval_logits/rejected": -2.8468399047851562,
      "eval_logps/chosen": -122.11509704589844,
      "eval_logps/rejected": -97.87715148925781,
      "eval_loss": 0.39711207151412964,
      "eval_rewards/accuracies": 0.7857142686843872,
      "eval_rewards/chosen": 1.1954243183135986,
      "eval_rewards/margins": 1.1722500324249268,
      "eval_rewards/rejected": 0.023174213245511055,
      "eval_runtime": 19.4446,
      "eval_samples_per_second": 1.44,
      "eval_steps_per_second": 1.44,
      "step": 440
    },
    {
      "epoch": 28.19,
      "grad_norm": 4.0338592529296875,
      "learning_rate": 4.721067777306581e-07,
      "logits/chosen": -2.6722593307495117,
      "logits/rejected": -2.682727575302124,
      "logps/chosen": -119.6982650756836,
      "logps/rejected": -105.52352905273438,
      "loss": 0.2108,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.5048611164093018,
      "rewards/margins": 1.8753631114959717,
      "rewards/rejected": -0.3705020546913147,
      "step": 444
    },
    {
      "epoch": 28.44,
      "grad_norm": 3.167048692703247,
      "learning_rate": 4.71585788522503e-07,
      "logits/chosen": -2.652644157409668,
      "logits/rejected": -2.6818065643310547,
      "logps/chosen": -113.42478942871094,
      "logps/rejected": -104.37007141113281,
      "loss": 0.2076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.474198341369629,
      "rewards/margins": 1.7973999977111816,
      "rewards/rejected": -0.32320165634155273,
      "step": 448
    },
    {
      "epoch": 28.7,
      "grad_norm": 3.4248368740081787,
      "learning_rate": 4.7106027168047833e-07,
      "logits/chosen": -2.7117884159088135,
      "logits/rejected": -2.77234148979187,
      "logps/chosen": -116.26769256591797,
      "logps/rejected": -101.22008514404297,
      "loss": 0.1858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5492609739303589,
      "rewards/margins": 1.9960644245147705,
      "rewards/rejected": -0.4468035101890564,
      "step": 452
    },
    {
      "epoch": 28.95,
      "grad_norm": 2.855328321456909,
      "learning_rate": 4.705302379424022e-07,
      "logits/chosen": -2.6717069149017334,
      "logits/rejected": -2.672755718231201,
      "logps/chosen": -116.75204467773438,
      "logps/rejected": -106.22608184814453,
      "loss": 0.2065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5143406391143799,
      "rewards/margins": 1.8520642518997192,
      "rewards/rejected": -0.33772367238998413,
      "step": 456
    },
    {
      "epoch": 29.21,
      "grad_norm": 2.8747594356536865,
      "learning_rate": 4.699956981383857e-07,
      "logits/chosen": -2.693591356277466,
      "logits/rejected": -2.662400960922241,
      "logps/chosen": -120.1160888671875,
      "logps/rejected": -91.72486877441406,
      "loss": 0.1874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6478428840637207,
      "rewards/margins": 2.0628483295440674,
      "rewards/rejected": -0.4150054454803467,
      "step": 460
    },
    {
      "epoch": 29.46,
      "grad_norm": 2.8170723915100098,
      "learning_rate": 4.694566631906116e-07,
      "logits/chosen": -2.6773173809051514,
      "logits/rejected": -2.672825574874878,
      "logps/chosen": -113.59075927734375,
      "logps/rejected": -103.25709533691406,
      "loss": 0.1984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4863029718399048,
      "rewards/margins": 1.892244577407837,
      "rewards/rejected": -0.405941367149353,
      "step": 464
    },
    {
      "epoch": 29.71,
      "grad_norm": 3.132080078125,
      "learning_rate": 4.689131441131118e-07,
      "logits/chosen": -2.6520447731018066,
      "logits/rejected": -2.6872832775115967,
      "logps/chosen": -118.07131958007812,
      "logps/rejected": -107.88346862792969,
      "loss": 0.2029,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.6008590459823608,
      "rewards/margins": 1.909461259841919,
      "rewards/rejected": -0.30860215425491333,
      "step": 468
    },
    {
      "epoch": 29.97,
      "grad_norm": 3.000974416732788,
      "learning_rate": 4.6836515201154135e-07,
      "logits/chosen": -2.709559202194214,
      "logits/rejected": -2.723712205886841,
      "logps/chosen": -112.09906768798828,
      "logps/rejected": -106.09660339355469,
      "loss": 0.2037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5191339254379272,
      "rewards/margins": 1.8524271249771118,
      "rewards/rejected": -0.3332933187484741,
      "step": 472
    },
    {
      "epoch": 30.22,
      "grad_norm": 2.544147491455078,
      "learning_rate": 4.678126980829524e-07,
      "logits/chosen": -2.678325653076172,
      "logits/rejected": -2.699833631515503,
      "logps/chosen": -115.4670639038086,
      "logps/rejected": -101.5027847290039,
      "loss": 0.1813,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4956222772598267,
      "rewards/margins": 1.9856685400009155,
      "rewards/rejected": -0.4900462329387665,
      "step": 476
    },
    {
      "epoch": 30.48,
      "grad_norm": 2.9770796298980713,
      "learning_rate": 4.672557936155649e-07,
      "logits/chosen": -2.708721160888672,
      "logits/rejected": -2.728306770324707,
      "logps/chosen": -109.71602630615234,
      "logps/rejected": -98.1273422241211,
      "loss": 0.1946,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.5404164791107178,
      "rewards/margins": 1.9511140584945679,
      "rewards/rejected": -0.4106977581977844,
      "step": 480
    },
    {
      "epoch": 30.48,
      "eval_logits/chosen": -2.764019012451172,
      "eval_logits/rejected": -2.8370797634124756,
      "eval_logps/chosen": -121.74720001220703,
      "eval_logps/rejected": -98.30638122558594,
      "eval_loss": 0.393496572971344,
      "eval_rewards/accuracies": 0.7857142686843872,
      "eval_rewards/chosen": 1.232213020324707,
      "eval_rewards/margins": 1.2519627809524536,
      "eval_rewards/rejected": -0.019749613478779793,
      "eval_runtime": 19.4183,
      "eval_samples_per_second": 1.442,
      "eval_steps_per_second": 1.442,
      "step": 480
    },
    {
      "epoch": 30.73,
      "grad_norm": 3.826716661453247,
      "learning_rate": 4.6669444998853605e-07,
      "logits/chosen": -2.6732892990112305,
      "logits/rejected": -2.700220823287964,
      "logps/chosen": -117.65553283691406,
      "logps/rejected": -111.19263458251953,
      "loss": 0.1876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6948100328445435,
      "rewards/margins": 2.0582737922668457,
      "rewards/rejected": -0.3634639382362366,
      "step": 484
    },
    {
      "epoch": 30.98,
      "grad_norm": 3.326704740524292,
      "learning_rate": 4.661286786717278e-07,
      "logits/chosen": -2.6515111923217773,
      "logits/rejected": -2.6504664421081543,
      "logps/chosen": -120.06623840332031,
      "logps/rejected": -101.47470092773438,
      "loss": 0.1828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5970648527145386,
      "rewards/margins": 2.0543787479400635,
      "rewards/rejected": -0.45731377601623535,
      "step": 488
    },
    {
      "epoch": 31.24,
      "grad_norm": 2.700209856033325,
      "learning_rate": 4.6555849122547263e-07,
      "logits/chosen": -2.6469173431396484,
      "logits/rejected": -2.666424036026001,
      "logps/chosen": -111.76505279541016,
      "logps/rejected": -103.40081024169922,
      "loss": 0.185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.524863839149475,
      "rewards/margins": 1.9786373376846313,
      "rewards/rejected": -0.4537736773490906,
      "step": 492
    },
    {
      "epoch": 31.49,
      "grad_norm": 2.592606544494629,
      "learning_rate": 4.649838993003372e-07,
      "logits/chosen": -2.7192447185516357,
      "logits/rejected": -2.726841926574707,
      "logps/chosen": -112.72746276855469,
      "logps/rejected": -100.51376342773438,
      "loss": 0.1678,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.5847443342208862,
      "rewards/margins": 2.0699594020843506,
      "rewards/rejected": -0.4852150082588196,
      "step": 496
    },
    {
      "epoch": 31.75,
      "grad_norm": 2.768076181411743,
      "learning_rate": 4.644049146368843e-07,
      "logits/chosen": -2.6624693870544434,
      "logits/rejected": -2.677476406097412,
      "logps/chosen": -119.13687896728516,
      "logps/rejected": -103.49778747558594,
      "loss": 0.1873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7677032947540283,
      "rewards/margins": 2.089017868041992,
      "rewards/rejected": -0.32131490111351013,
      "step": 500
    },
    {
      "epoch": 32.0,
      "grad_norm": 2.4647607803344727,
      "learning_rate": 4.638215490654329e-07,
      "logits/chosen": -2.6946330070495605,
      "logits/rejected": -2.6913187503814697,
      "logps/chosen": -118.64775085449219,
      "logps/rejected": -107.40900421142578,
      "loss": 0.1847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5691322088241577,
      "rewards/margins": 2.105942964553833,
      "rewards/rejected": -0.5368106365203857,
      "step": 504
    },
    {
      "epoch": 32.25,
      "grad_norm": 3.723209857940674,
      "learning_rate": 4.6323381450581665e-07,
      "logits/chosen": -2.6984715461730957,
      "logits/rejected": -2.686072826385498,
      "logps/chosen": -118.19576263427734,
      "logps/rejected": -104.87760162353516,
      "loss": 0.1853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.504220724105835,
      "rewards/margins": 2.023249864578247,
      "rewards/rejected": -0.519028902053833,
      "step": 508
    },
    {
      "epoch": 32.51,
      "grad_norm": 2.6810081005096436,
      "learning_rate": 4.6264172296714e-07,
      "logits/chosen": -2.6358213424682617,
      "logits/rejected": -2.6452157497406006,
      "logps/chosen": -116.47501373291016,
      "logps/rejected": -104.9029769897461,
      "loss": 0.1948,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.6027023792266846,
      "rewards/margins": 2.105417251586914,
      "rewards/rejected": -0.5027150511741638,
      "step": 512
    },
    {
      "epoch": 32.76,
      "grad_norm": 2.1455435752868652,
      "learning_rate": 4.6204528654753304e-07,
      "logits/chosen": -2.692194700241089,
      "logits/rejected": -2.705289125442505,
      "logps/chosen": -113.38731384277344,
      "logps/rejected": -103.49237823486328,
      "loss": 0.1617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7136329412460327,
      "rewards/margins": 2.1900761127471924,
      "rewards/rejected": -0.47644317150115967,
      "step": 516
    },
    {
      "epoch": 33.02,
      "grad_norm": 3.499128818511963,
      "learning_rate": 4.614445174339045e-07,
      "logits/chosen": -2.661409616470337,
      "logits/rejected": -2.700990915298462,
      "logps/chosen": -113.71514892578125,
      "logps/rejected": -103.91769409179688,
      "loss": 0.1507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7427312135696411,
      "rewards/margins": 2.1958351135253906,
      "rewards/rejected": -0.45310381054878235,
      "step": 520
    },
    {
      "epoch": 33.02,
      "eval_logits/chosen": -2.755352020263672,
      "eval_logits/rejected": -2.826873540878296,
      "eval_logps/chosen": -121.47989654541016,
      "eval_logps/rejected": -98.8564453125,
      "eval_loss": 0.3899853527545929,
      "eval_rewards/accuracies": 0.7857142686843872,
      "eval_rewards/chosen": 1.2589442729949951,
      "eval_rewards/margins": 1.333700180053711,
      "eval_rewards/rejected": -0.07475597411394119,
      "eval_runtime": 19.3899,
      "eval_samples_per_second": 1.444,
      "eval_steps_per_second": 1.444,
      "step": 520
    },
    {
      "epoch": 33.27,
      "grad_norm": 2.419506549835205,
      "learning_rate": 4.6083942790169206e-07,
      "logits/chosen": -2.6735386848449707,
      "logits/rejected": -2.696384906768799,
      "logps/chosen": -111.79097747802734,
      "logps/rejected": -98.7368392944336,
      "loss": 0.1581,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.5749679803848267,
      "rewards/margins": 2.265718460083008,
      "rewards/rejected": -0.690750777721405,
      "step": 524
    },
    {
      "epoch": 33.52,
      "grad_norm": 2.1046078205108643,
      "learning_rate": 4.6023003031461227e-07,
      "logits/chosen": -2.6465351581573486,
      "logits/rejected": -2.6622884273529053,
      "logps/chosen": -114.73282623291016,
      "logps/rejected": -103.12973022460938,
      "loss": 0.1778,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6710339784622192,
      "rewards/margins": 2.119300365447998,
      "rewards/rejected": -0.4482663571834564,
      "step": 528
    },
    {
      "epoch": 33.78,
      "grad_norm": 3.0457003116607666,
      "learning_rate": 4.596163371244076e-07,
      "logits/chosen": -2.6422927379608154,
      "logits/rejected": -2.6462604999542236,
      "logps/chosen": -114.708740234375,
      "logps/rejected": -105.07784271240234,
      "loss": 0.16,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7936890125274658,
      "rewards/margins": 2.2034826278686523,
      "rewards/rejected": -0.40979355573654175,
      "step": 532
    },
    {
      "epoch": 34.03,
      "grad_norm": 2.8257133960723877,
      "learning_rate": 4.5899836087059174e-07,
      "logits/chosen": -2.6918063163757324,
      "logits/rejected": -2.696005344390869,
      "logps/chosen": -117.54481506347656,
      "logps/rejected": -111.74267578125,
      "loss": 0.1753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.625847339630127,
      "rewards/margins": 2.2450764179229736,
      "rewards/rejected": -0.6192291975021362,
      "step": 536
    },
    {
      "epoch": 34.29,
      "grad_norm": 2.5381155014038086,
      "learning_rate": 4.58376114180194e-07,
      "logits/chosen": -2.684847116470337,
      "logits/rejected": -2.6875247955322266,
      "logps/chosen": -121.1004867553711,
      "logps/rejected": -101.69570922851562,
      "loss": 0.1562,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.6668047904968262,
      "rewards/margins": 2.3632285594940186,
      "rewards/rejected": -0.6964237689971924,
      "step": 540
    },
    {
      "epoch": 34.54,
      "grad_norm": 2.519352912902832,
      "learning_rate": 4.577496097675009e-07,
      "logits/chosen": -2.6664843559265137,
      "logits/rejected": -2.681034803390503,
      "logps/chosen": -115.94261169433594,
      "logps/rejected": -108.1679458618164,
      "loss": 0.1516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7702218294143677,
      "rewards/margins": 2.2215099334716797,
      "rewards/rejected": -0.4512881338596344,
      "step": 544
    },
    {
      "epoch": 34.79,
      "grad_norm": 2.789398431777954,
      "learning_rate": 4.571188604337962e-07,
      "logits/chosen": -2.668377637863159,
      "logits/rejected": -2.688797950744629,
      "logps/chosen": -108.2844467163086,
      "logps/rejected": -104.94053649902344,
      "loss": 0.1838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.559061050415039,
      "rewards/margins": 2.064011812210083,
      "rewards/rejected": -0.504950761795044,
      "step": 548
    },
    {
      "epoch": 35.05,
      "grad_norm": 3.570829153060913,
      "learning_rate": 4.5648387906709995e-07,
      "logits/chosen": -2.6559903621673584,
      "logits/rejected": -2.650508165359497,
      "logps/chosen": -112.94985961914062,
      "logps/rejected": -101.36627197265625,
      "loss": 0.1484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.709464192390442,
      "rewards/margins": 2.303464889526367,
      "rewards/rejected": -0.5940006375312805,
      "step": 552
    },
    {
      "epoch": 35.3,
      "grad_norm": 2.487671136856079,
      "learning_rate": 4.558446786419045e-07,
      "logits/chosen": -2.6454193592071533,
      "logits/rejected": -2.667191982269287,
      "logps/chosen": -112.91888427734375,
      "logps/rejected": -100.34318542480469,
      "loss": 0.1571,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.6041696071624756,
      "rewards/margins": 2.3108744621276855,
      "rewards/rejected": -0.7067050337791443,
      "step": 556
    },
    {
      "epoch": 35.56,
      "grad_norm": 2.396897315979004,
      "learning_rate": 4.552012722189099e-07,
      "logits/chosen": -2.6444239616394043,
      "logits/rejected": -2.6464180946350098,
      "logps/chosen": -115.7559814453125,
      "logps/rejected": -111.63291931152344,
      "loss": 0.1752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7378344535827637,
      "rewards/margins": 2.1783785820007324,
      "rewards/rejected": -0.4405441880226135,
      "step": 560
    },
    {
      "epoch": 35.56,
      "eval_logits/chosen": -2.746135711669922,
      "eval_logits/rejected": -2.8161184787750244,
      "eval_logps/chosen": -121.27652740478516,
      "eval_logps/rejected": -99.45245361328125,
      "eval_loss": 0.38749733567237854,
      "eval_rewards/accuracies": 0.7857142686843872,
      "eval_rewards/chosen": 1.2792812585830688,
      "eval_rewards/margins": 1.4136368036270142,
      "eval_rewards/rejected": -0.1343555599451065,
      "eval_runtime": 19.3831,
      "eval_samples_per_second": 1.445,
      "eval_steps_per_second": 1.445,
      "step": 560
    }
  ],
  "logging_steps": 4,
  "max_steps": 2800,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 187,
  "save_steps": 40,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}