{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9865871833084947,
  "eval_steps": 500,
  "global_step": 501,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.029806259314456036,
      "grad_norm": 1823.4444580078125,
      "learning_rate": 2.5000000000000004e-07,
      "log_odds_chosen": -0.22316808998584747,
      "log_odds_ratio": -1.008633017539978,
      "logits/chosen": 204.3894805908203,
      "logits/rejected": 203.0862274169922,
      "logps/chosen": -14.825190544128418,
      "logps/rejected": -14.602023124694824,
      "loss": 15.3951,
      "nll_loss": 14.544248580932617,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -7.412595272064209,
      "rewards/margins": -0.1115839034318924,
      "rewards/rejected": -7.301011562347412,
      "step": 5
    },
    {
      "epoch": 0.05961251862891207,
      "grad_norm": 1235.2249755859375,
      "learning_rate": 5.000000000000001e-07,
      "log_odds_chosen": 0.2599984109401703,
      "log_odds_ratio": -0.7687832713127136,
      "logits/chosen": 219.47842407226562,
      "logits/rejected": 223.53195190429688,
      "logps/chosen": -12.257360458374023,
      "logps/rejected": -12.516705513000488,
      "loss": 13.0935,
      "nll_loss": 12.36180305480957,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -6.128680229187012,
      "rewards/margins": 0.1296723335981369,
      "rewards/rejected": -6.258352756500244,
      "step": 10
    },
    {
      "epoch": 0.08941877794336811,
      "grad_norm": 800.8740844726562,
      "learning_rate": 7.5e-07,
      "log_odds_chosen": 0.07262952625751495,
      "log_odds_ratio": -0.7626513242721558,
      "logits/chosen": 284.1068115234375,
      "logits/rejected": 263.359619140625,
      "logps/chosen": -8.042943000793457,
      "logps/rejected": -8.115506172180176,
      "loss": 8.8161,
      "nll_loss": 8.040520668029785,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -4.0214715003967285,
      "rewards/margins": 0.03628172725439072,
      "rewards/rejected": -4.057753086090088,
      "step": 15
    },
    {
      "epoch": 0.11922503725782414,
      "grad_norm": 245.36756896972656,
      "learning_rate": 1.0000000000000002e-06,
      "log_odds_chosen": -0.1374012976884842,
      "log_odds_ratio": -0.9441366195678711,
      "logits/chosen": 284.14410400390625,
      "logits/rejected": 279.113037109375,
      "logps/chosen": -5.502886772155762,
      "logps/rejected": -5.366313934326172,
      "loss": 5.9511,
      "nll_loss": 5.569659233093262,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -2.751443386077881,
      "rewards/margins": -0.06828644871711731,
      "rewards/rejected": -2.683156967163086,
      "step": 20
    },
    {
      "epoch": 0.14903129657228018,
      "grad_norm": 150.3289337158203,
      "learning_rate": 1.25e-06,
      "log_odds_chosen": -0.0038304091431200504,
      "log_odds_ratio": -0.8650141954421997,
      "logits/chosen": 296.8591003417969,
      "logits/rejected": 306.09613037109375,
      "logps/chosen": -3.4051125049591064,
      "logps/rejected": -3.383711338043213,
      "loss": 3.9954,
      "nll_loss": 3.5142741203308105,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.7025562524795532,
      "rewards/margins": -0.010700652375817299,
      "rewards/rejected": -1.6918556690216064,
      "step": 25
    },
    {
      "epoch": 0.17883755588673622,
      "grad_norm": 95.30947875976562,
      "learning_rate": 1.5e-06,
      "log_odds_chosen": -0.02694419026374817,
      "log_odds_ratio": -0.8674762845039368,
      "logits/chosen": 341.46405029296875,
      "logits/rejected": 369.68597412109375,
      "logps/chosen": -2.748025894165039,
      "logps/rejected": -2.6941990852355957,
      "loss": 3.0003,
      "nll_loss": 2.7433438301086426,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.3740129470825195,
      "rewards/margins": -0.02691327966749668,
      "rewards/rejected": -1.3470995426177979,
      "step": 30
    },
    {
      "epoch": 0.20864381520119224,
      "grad_norm": 61.000022888183594,
      "learning_rate": 1.75e-06,
      "log_odds_chosen": 0.20579464733600616,
      "log_odds_ratio": -0.7235999703407288,
      "logits/chosen": 380.79266357421875,
      "logits/rejected": 368.7811279296875,
      "logps/chosen": -1.8658641576766968,
      "logps/rejected": -2.010704517364502,
      "loss": 2.4897,
      "nll_loss": 2.063866376876831,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.9329320788383484,
      "rewards/margins": 0.07242023944854736,
      "rewards/rejected": -1.005352258682251,
      "step": 35
    },
    {
      "epoch": 0.23845007451564829,
      "grad_norm": 52.54775619506836,
      "learning_rate": 2.0000000000000003e-06,
      "log_odds_chosen": 0.14920620620250702,
      "log_odds_ratio": -0.7215272188186646,
      "logits/chosen": 373.4493713378906,
      "logits/rejected": 372.3648376464844,
      "logps/chosen": -1.7074140310287476,
      "logps/rejected": -1.8317168951034546,
      "loss": 2.3294,
      "nll_loss": 2.0935075283050537,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.8537070155143738,
      "rewards/margins": 0.06215135008096695,
      "rewards/rejected": -0.9158584475517273,
      "step": 40
    },
    {
      "epoch": 0.26825633383010433,
      "grad_norm": 58.8853645324707,
      "learning_rate": 2.25e-06,
      "log_odds_chosen": 0.20898687839508057,
      "log_odds_ratio": -0.666365385055542,
      "logits/chosen": 390.11090087890625,
      "logits/rejected": 399.29052734375,
      "logps/chosen": -1.6940256357192993,
      "logps/rejected": -1.859763503074646,
      "loss": 2.2384,
      "nll_loss": 1.81049382686615,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8470128178596497,
      "rewards/margins": 0.08286889642477036,
      "rewards/rejected": -0.929881751537323,
      "step": 45
    },
    {
      "epoch": 0.29806259314456035,
      "grad_norm": 53.57241439819336,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 0.4837431013584137,
      "log_odds_ratio": -0.5640600919723511,
      "logits/chosen": 397.261474609375,
      "logits/rejected": 418.50787353515625,
      "logps/chosen": -1.548592209815979,
      "logps/rejected": -1.9497201442718506,
      "loss": 2.2689,
      "nll_loss": 1.9370243549346924,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7742961049079895,
      "rewards/margins": 0.20056398212909698,
      "rewards/rejected": -0.9748600721359253,
      "step": 50
    },
    {
      "epoch": 0.32786885245901637,
      "grad_norm": 54.863670349121094,
      "learning_rate": 2.7500000000000004e-06,
      "log_odds_chosen": 0.10273619741201401,
      "log_odds_ratio": -0.7500853538513184,
      "logits/chosen": 393.6875,
      "logits/rejected": 385.71246337890625,
      "logps/chosen": -1.6378755569458008,
      "logps/rejected": -1.7148265838623047,
      "loss": 2.343,
      "nll_loss": 2.059002637863159,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.8189377784729004,
      "rewards/margins": 0.038475506007671356,
      "rewards/rejected": -0.8574132919311523,
      "step": 55
    },
    {
      "epoch": 0.35767511177347244,
      "grad_norm": 52.10759735107422,
      "learning_rate": 3e-06,
      "log_odds_chosen": 0.20460805296897888,
      "log_odds_ratio": -0.6378257870674133,
      "logits/chosen": 393.1427917480469,
      "logits/rejected": 384.66424560546875,
      "logps/chosen": -1.5411105155944824,
      "logps/rejected": -1.693114995956421,
      "loss": 2.0643,
      "nll_loss": 1.8742763996124268,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7705552577972412,
      "rewards/margins": 0.07600229978561401,
      "rewards/rejected": -0.8465574979782104,
      "step": 60
    },
    {
      "epoch": 0.38748137108792846,
      "grad_norm": 36.03993225097656,
      "learning_rate": 3.2500000000000002e-06,
      "log_odds_chosen": 0.31621289253234863,
      "log_odds_ratio": -0.6504405736923218,
      "logits/chosen": 391.6888427734375,
      "logits/rejected": 392.4349365234375,
      "logps/chosen": -1.3424034118652344,
      "logps/rejected": -1.5777828693389893,
      "loss": 1.9359,
      "nll_loss": 1.5658212900161743,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6712017059326172,
      "rewards/margins": 0.11768964678049088,
      "rewards/rejected": -0.7888914346694946,
      "step": 65
    },
    {
      "epoch": 0.4172876304023845,
      "grad_norm": 41.6033821105957,
      "learning_rate": 3.5e-06,
      "log_odds_chosen": 0.3797632157802582,
      "log_odds_ratio": -0.6185951828956604,
      "logits/chosen": 376.75579833984375,
      "logits/rejected": 385.7332763671875,
      "logps/chosen": -1.507440209388733,
      "logps/rejected": -1.8190501928329468,
      "loss": 2.0024,
      "nll_loss": 1.7485978603363037,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7537201046943665,
      "rewards/margins": 0.1558050811290741,
      "rewards/rejected": -0.9095250964164734,
      "step": 70
    },
    {
      "epoch": 0.44709388971684055,
      "grad_norm": 49.81876754760742,
      "learning_rate": 3.7500000000000005e-06,
      "log_odds_chosen": 0.5595852732658386,
      "log_odds_ratio": -0.531262218952179,
      "logits/chosen": 394.0195617675781,
      "logits/rejected": 380.848388671875,
      "logps/chosen": -1.278014898300171,
      "logps/rejected": -1.7234604358673096,
      "loss": 1.8859,
      "nll_loss": 1.5626357793807983,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.6390074491500854,
      "rewards/margins": 0.22272281348705292,
      "rewards/rejected": -0.8617302179336548,
      "step": 75
    },
    {
      "epoch": 0.47690014903129657,
      "grad_norm": 49.527286529541016,
      "learning_rate": 4.000000000000001e-06,
      "log_odds_chosen": 0.4827095866203308,
      "log_odds_ratio": -0.5827818512916565,
      "logits/chosen": 383.16455078125,
      "logits/rejected": 405.3050231933594,
      "logps/chosen": -1.4663052558898926,
      "logps/rejected": -1.8465086221694946,
      "loss": 1.9594,
      "nll_loss": 1.6925424337387085,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7331526279449463,
      "rewards/margins": 0.19010166823863983,
      "rewards/rejected": -0.9232543110847473,
      "step": 80
    },
    {
      "epoch": 0.5067064083457526,
      "grad_norm": 35.50678634643555,
      "learning_rate": 4.25e-06,
      "log_odds_chosen": 0.32843467593193054,
      "log_odds_ratio": -0.7204899787902832,
      "logits/chosen": 409.7994689941406,
      "logits/rejected": 394.08575439453125,
      "logps/chosen": -1.5447129011154175,
      "logps/rejected": -1.7741330862045288,
      "loss": 1.9717,
      "nll_loss": 1.7773849964141846,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7723564505577087,
      "rewards/margins": 0.11471016705036163,
      "rewards/rejected": -0.8870665431022644,
      "step": 85
    },
    {
      "epoch": 0.5365126676602087,
      "grad_norm": 32.97898864746094,
      "learning_rate": 4.5e-06,
      "log_odds_chosen": 1.082437515258789,
      "log_odds_ratio": -0.48028555512428284,
      "logits/chosen": 400.08160400390625,
      "logits/rejected": 415.0059509277344,
      "logps/chosen": -1.4097836017608643,
      "logps/rejected": -2.338646411895752,
      "loss": 1.7591,
      "nll_loss": 1.5575084686279297,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7048918008804321,
      "rewards/margins": 0.4644315242767334,
      "rewards/rejected": -1.169323205947876,
      "step": 90
    },
    {
      "epoch": 0.5663189269746647,
      "grad_norm": 59.56174850463867,
      "learning_rate": 4.75e-06,
      "log_odds_chosen": 0.43072813749313354,
      "log_odds_ratio": -0.5813931226730347,
      "logits/chosen": 370.78253173828125,
      "logits/rejected": 383.03009033203125,
      "logps/chosen": -1.1654596328735352,
      "logps/rejected": -1.4328858852386475,
      "loss": 1.8455,
      "nll_loss": 1.4887888431549072,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5827298164367676,
      "rewards/margins": 0.13371312618255615,
      "rewards/rejected": -0.7164429426193237,
      "step": 95
    },
    {
      "epoch": 0.5961251862891207,
      "grad_norm": 98.94036102294922,
      "learning_rate": 5e-06,
      "log_odds_chosen": 0.6077507138252258,
      "log_odds_ratio": -0.5442172884941101,
      "logits/chosen": 406.62945556640625,
      "logits/rejected": 442.9092712402344,
      "logps/chosen": -1.3020541667938232,
      "logps/rejected": -1.8070911169052124,
      "loss": 1.8887,
      "nll_loss": 1.6196590662002563,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6510270833969116,
      "rewards/margins": 0.2525184154510498,
      "rewards/rejected": -0.9035455584526062,
      "step": 100
    },
    {
      "epoch": 0.6259314456035767,
      "grad_norm": 64.72420501708984,
      "learning_rate": 4.8795003647426654e-06,
      "log_odds_chosen": 0.6622194051742554,
      "log_odds_ratio": -0.5436784625053406,
      "logits/chosen": 396.576416015625,
      "logits/rejected": 409.37408447265625,
      "logps/chosen": -1.239157795906067,
      "logps/rejected": -1.7216756343841553,
      "loss": 2.0086,
      "nll_loss": 1.743311882019043,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6195788979530334,
      "rewards/margins": 0.24125893414020538,
      "rewards/rejected": -0.8608378171920776,
      "step": 105
    },
    {
      "epoch": 0.6557377049180327,
      "grad_norm": 156.0756072998047,
      "learning_rate": 4.767312946227961e-06,
      "log_odds_chosen": 0.552756667137146,
      "log_odds_ratio": -0.5492970943450928,
      "logits/chosen": 376.5224304199219,
      "logits/rejected": 374.7490234375,
      "logps/chosen": -1.258983850479126,
      "logps/rejected": -1.6897951364517212,
      "loss": 1.8644,
      "nll_loss": 1.6051260232925415,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.629491925239563,
      "rewards/margins": 0.21540567278862,
      "rewards/rejected": -0.8448975682258606,
      "step": 110
    },
    {
      "epoch": 0.6855439642324889,
      "grad_norm": 27.666635513305664,
      "learning_rate": 4.662524041201569e-06,
      "log_odds_chosen": 0.7742821574211121,
      "log_odds_ratio": -0.4874287545681,
      "logits/chosen": 401.1915283203125,
      "logits/rejected": 406.65716552734375,
      "logps/chosen": -1.1674349308013916,
      "logps/rejected": -1.7199904918670654,
      "loss": 1.8546,
      "nll_loss": 1.5520570278167725,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.5837174654006958,
      "rewards/margins": 0.2762778401374817,
      "rewards/rejected": -0.8599952459335327,
      "step": 115
    },
    {
      "epoch": 0.7153502235469449,
      "grad_norm": 39.84739303588867,
      "learning_rate": 4.564354645876385e-06,
      "log_odds_chosen": 0.7211123108863831,
      "log_odds_ratio": -0.5134823322296143,
      "logits/chosen": 388.77838134765625,
      "logits/rejected": 388.0323181152344,
      "logps/chosen": -1.1216659545898438,
      "logps/rejected": -1.614723801612854,
      "loss": 1.8847,
      "nll_loss": 1.5851175785064697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5608329772949219,
      "rewards/margins": 0.24652889370918274,
      "rewards/rejected": -0.807361900806427,
      "step": 120
    },
    {
      "epoch": 0.7451564828614009,
      "grad_norm": 29.12469482421875,
      "learning_rate": 4.47213595499958e-06,
      "log_odds_chosen": 0.3196253776550293,
      "log_odds_ratio": -0.6582637429237366,
      "logits/chosen": 386.79583740234375,
      "logits/rejected": 398.7740783691406,
      "logps/chosen": -1.351342797279358,
      "logps/rejected": -1.5734379291534424,
      "loss": 1.8179,
      "nll_loss": 1.5471035242080688,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.675671398639679,
      "rewards/margins": 0.11104752123355865,
      "rewards/rejected": -0.7867189645767212,
      "step": 125
    },
    {
      "epoch": 0.7749627421758569,
      "grad_norm": 38.485130310058594,
      "learning_rate": 4.385290096535147e-06,
      "log_odds_chosen": 0.6995328068733215,
      "log_odds_ratio": -0.5530967712402344,
      "logits/chosen": 409.95849609375,
      "logits/rejected": 397.23114013671875,
      "logps/chosen": -1.1655128002166748,
      "logps/rejected": -1.6789804697036743,
      "loss": 1.8506,
      "nll_loss": 1.5426290035247803,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5827564001083374,
      "rewards/margins": 0.2567337453365326,
      "rewards/rejected": -0.8394902348518372,
      "step": 130
    },
    {
      "epoch": 0.8047690014903129,
      "grad_norm": 52.03726577758789,
      "learning_rate": 4.303314829119352e-06,
      "log_odds_chosen": 0.42176657915115356,
      "log_odds_ratio": -0.6203495264053345,
      "logits/chosen": 415.891357421875,
      "logits/rejected": 418.27984619140625,
      "logps/chosen": -1.4386600255966187,
      "logps/rejected": -1.7602298259735107,
      "loss": 1.873,
      "nll_loss": 1.672498106956482,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7193300127983093,
      "rewards/margins": 0.16078472137451172,
      "rewards/rejected": -0.8801149129867554,
      "step": 135
    },
    {
      "epoch": 0.834575260804769,
      "grad_norm": 37.35191345214844,
      "learning_rate": 4.2257712736425835e-06,
      "log_odds_chosen": 0.30007433891296387,
      "log_odds_ratio": -0.659325897693634,
      "logits/chosen": 397.5603942871094,
      "logits/rejected": 403.28192138671875,
      "logps/chosen": -1.314620852470398,
      "logps/rejected": -1.552886962890625,
      "loss": 1.8804,
      "nll_loss": 1.7158609628677368,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.657310426235199,
      "rewards/margins": 0.11913303285837173,
      "rewards/rejected": -0.7764434814453125,
      "step": 140
    },
    {
      "epoch": 0.8643815201192251,
      "grad_norm": 40.69832992553711,
      "learning_rate": 4.1522739926869985e-06,
      "log_odds_chosen": 0.15929968655109406,
      "log_odds_ratio": -0.6508444547653198,
      "logits/chosen": 392.15386962890625,
      "logits/rejected": 395.3309631347656,
      "logps/chosen": -1.2985050678253174,
      "logps/rejected": -1.4000908136367798,
      "loss": 1.8468,
      "nll_loss": 1.581002950668335,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6492525339126587,
      "rewards/margins": 0.0507928729057312,
      "rewards/rejected": -0.7000454068183899,
      "step": 145
    },
    {
      "epoch": 0.8941877794336811,
      "grad_norm": 33.805145263671875,
      "learning_rate": 4.082482904638631e-06,
      "log_odds_chosen": 0.7655292749404907,
      "log_odds_ratio": -0.514534592628479,
      "logits/chosen": 398.47711181640625,
      "logits/rejected": 415.9359436035156,
      "logps/chosen": -1.1850507259368896,
      "logps/rejected": -1.7938915491104126,
      "loss": 1.7734,
      "nll_loss": 1.4344916343688965,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.5925253629684448,
      "rewards/margins": 0.3044203519821167,
      "rewards/rejected": -0.8969457745552063,
      "step": 150
    },
    {
      "epoch": 0.9239940387481371,
      "grad_norm": 21.895387649536133,
      "learning_rate": 4.016096644512495e-06,
      "log_odds_chosen": 0.26020118594169617,
      "log_odds_ratio": -0.6326924562454224,
      "logits/chosen": 378.7313537597656,
      "logits/rejected": 393.6383361816406,
      "logps/chosen": -1.2521542310714722,
      "logps/rejected": -1.4581738710403442,
      "loss": 1.7575,
      "nll_loss": 1.3882125616073608,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6260771155357361,
      "rewards/margins": 0.10300980508327484,
      "rewards/rejected": -0.7290869355201721,
      "step": 155
    },
    {
      "epoch": 0.9538002980625931,
      "grad_norm": 65.63549041748047,
      "learning_rate": 3.952847075210474e-06,
      "log_odds_chosen": 0.28196483850479126,
      "log_odds_ratio": -0.6715171337127686,
      "logits/chosen": 386.80755615234375,
      "logits/rejected": 431.31146240234375,
      "logps/chosen": -1.1752804517745972,
      "logps/rejected": -1.395171880722046,
      "loss": 1.7333,
      "nll_loss": 1.422628402709961,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.5876402258872986,
      "rewards/margins": 0.1099456176161766,
      "rewards/rejected": -0.697585940361023,
      "step": 160
    },
    {
      "epoch": 0.9836065573770492,
      "grad_norm": 31.469694137573242,
      "learning_rate": 3.892494720807615e-06,
      "log_odds_chosen": 0.20551732182502747,
      "log_odds_ratio": -0.6562041640281677,
      "logits/chosen": 399.196533203125,
      "logits/rejected": 412.1044006347656,
      "logps/chosen": -1.233426809310913,
      "logps/rejected": -1.369274377822876,
      "loss": 1.8238,
      "nll_loss": 1.516705870628357,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6167134046554565,
      "rewards/margins": 0.06792382895946503,
      "rewards/rejected": -0.684637188911438,
      "step": 165
    },
    {
      "epoch": 0.9955290611028316,
      "eval_log_odds_chosen": 0.2615682780742645,
      "eval_log_odds_ratio": -0.6631397604942322,
      "eval_logits/chosen": 322.6454162597656,
      "eval_logits/rejected": 293.2744445800781,
      "eval_logps/chosen": -1.0755056142807007,
      "eval_logps/rejected": -1.263703465461731,
      "eval_loss": 1.8175820112228394,
      "eval_nll_loss": 1.4782638549804688,
      "eval_rewards/accuracies": 0.5467625856399536,
      "eval_rewards/chosen": -0.5377528071403503,
      "eval_rewards/margins": 0.09409892559051514,
      "eval_rewards/rejected": -0.6318517327308655,
      "eval_runtime": 351.5652,
      "eval_samples_per_second": 1.573,
      "eval_steps_per_second": 0.395,
      "step": 167
    },
    {
      "epoch": 1.0134128166915053,
      "grad_norm": 19.610740661621094,
      "learning_rate": 3.834824944236852e-06,
      "log_odds_chosen": 0.7168810963630676,
      "log_odds_ratio": -0.48998212814331055,
      "logits/chosen": 379.9788513183594,
      "logits/rejected": 404.4870910644531,
      "logps/chosen": -1.0245921611785889,
      "logps/rejected": -1.525979995727539,
      "loss": 1.6186,
      "nll_loss": 1.2430717945098877,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.5122960805892944,
      "rewards/margins": 0.2506938874721527,
      "rewards/rejected": -0.7629899978637695,
      "step": 170
    },
    {
      "epoch": 1.0432190760059612,
      "grad_norm": 27.745262145996094,
      "learning_rate": 3.7796447300922724e-06,
      "log_odds_chosen": 1.422102689743042,
      "log_odds_ratio": -0.3321012854576111,
      "logits/chosen": 357.04718017578125,
      "logits/rejected": 398.559814453125,
      "logps/chosen": -0.7521129846572876,
      "logps/rejected": -1.7101812362670898,
      "loss": 1.286,
      "nll_loss": 1.213651418685913,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.3760564923286438,
      "rewards/margins": 0.47903409600257874,
      "rewards/rejected": -0.8550906181335449,
      "step": 175
    },
    {
      "epoch": 1.0730253353204173,
      "grad_norm": 20.82083511352539,
      "learning_rate": 3.72677996249965e-06,
      "log_odds_chosen": 1.3344449996948242,
      "log_odds_ratio": -0.3116641640663147,
      "logits/chosen": 356.35601806640625,
      "logits/rejected": 331.2690124511719,
      "logps/chosen": -0.8246381878852844,
      "logps/rejected": -1.7447946071624756,
      "loss": 1.3455,
      "nll_loss": 1.2647905349731445,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.4123190939426422,
      "rewards/margins": 0.46007823944091797,
      "rewards/rejected": -0.8723973035812378,
      "step": 180
    },
    {
      "epoch": 1.1028315946348732,
      "grad_norm": 33.03415298461914,
      "learning_rate": 3.6760731104690393e-06,
      "log_odds_chosen": 1.7977418899536133,
      "log_odds_ratio": -0.29093313217163086,
      "logits/chosen": 388.28021240234375,
      "logits/rejected": 380.4255065917969,
      "logps/chosen": -0.8684544563293457,
      "logps/rejected": -2.099797487258911,
      "loss": 1.2411,
      "nll_loss": 1.0944446325302124,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.43422722816467285,
      "rewards/margins": 0.6156715750694275,
      "rewards/rejected": -1.0498987436294556,
      "step": 185
    },
    {
      "epoch": 1.1326378539493294,
      "grad_norm": 48.01770782470703,
      "learning_rate": 3.6273812505500587e-06,
      "log_odds_chosen": 1.2517069578170776,
      "log_odds_ratio": -0.3731076717376709,
      "logits/chosen": 356.5531005859375,
      "logits/rejected": 402.0587463378906,
      "logps/chosen": -0.8801576495170593,
      "logps/rejected": -1.7424513101577759,
      "loss": 1.326,
      "nll_loss": 1.140998125076294,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.44007882475852966,
      "rewards/margins": 0.4311468005180359,
      "rewards/rejected": -0.8712256550788879,
      "step": 190
    },
    {
      "epoch": 1.1624441132637853,
      "grad_norm": 25.167158126831055,
      "learning_rate": 3.5805743701971648e-06,
      "log_odds_chosen": 1.9819135665893555,
      "log_odds_ratio": -0.29898396134376526,
      "logits/chosen": 382.34100341796875,
      "logits/rejected": 395.79425048828125,
      "logps/chosen": -0.9413985013961792,
      "logps/rejected": -2.448883533477783,
      "loss": 1.3195,
      "nll_loss": 1.2245818376541138,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4706992506980896,
      "rewards/margins": 0.7537423968315125,
      "rewards/rejected": -1.2244417667388916,
      "step": 195
    },
    {
      "epoch": 1.1922503725782414,
      "grad_norm": 29.544389724731445,
      "learning_rate": 3.5355339059327378e-06,
      "log_odds_chosen": 2.2445785999298096,
      "log_odds_ratio": -0.27407315373420715,
      "logits/chosen": 399.3606262207031,
      "logits/rejected": 381.97125244140625,
      "logps/chosen": -0.7390884757041931,
      "logps/rejected": -2.445159435272217,
      "loss": 1.3027,
      "nll_loss": 1.0836920738220215,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.36954423785209656,
      "rewards/margins": 0.853035569190979,
      "rewards/rejected": -1.2225797176361084,
      "step": 200
    },
    {
      "epoch": 1.2220566318926975,
      "grad_norm": 33.41192626953125,
      "learning_rate": 3.4921514788478916e-06,
      "log_odds_chosen": 2.2366576194763184,
      "log_odds_ratio": -0.2573709785938263,
      "logits/chosen": 359.3567810058594,
      "logits/rejected": 353.2535400390625,
      "logps/chosen": -0.7408849596977234,
      "logps/rejected": -2.3824286460876465,
      "loss": 1.2558,
      "nll_loss": 1.1112185716629028,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3704424798488617,
      "rewards/margins": 0.8207718729972839,
      "rewards/rejected": -1.1912143230438232,
      "step": 205
    },
    {
      "epoch": 1.2518628912071534,
      "grad_norm": 18.833148956298828,
      "learning_rate": 3.450327796711771e-06,
      "log_odds_chosen": 2.773773431777954,
      "log_odds_ratio": -0.24581894278526306,
      "logits/chosen": 363.11090087890625,
      "logits/rejected": 392.077880859375,
      "logps/chosen": -0.7906020879745483,
      "logps/rejected": -2.9855308532714844,
      "loss": 1.2167,
      "nll_loss": 1.0773674249649048,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.39530104398727417,
      "rewards/margins": 1.0974645614624023,
      "rewards/rejected": -1.4927654266357422,
      "step": 210
    },
    {
      "epoch": 1.2816691505216096,
      "grad_norm": 32.97148895263672,
      "learning_rate": 3.409971697352368e-06,
      "log_odds_chosen": 2.409407138824463,
      "log_odds_ratio": -0.2513101100921631,
      "logits/chosen": 386.68804931640625,
      "logits/rejected": 369.2494812011719,
      "logps/chosen": -0.7668555378913879,
      "logps/rejected": -2.570010185241699,
      "loss": 1.232,
      "nll_loss": 1.1109261512756348,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.38342776894569397,
      "rewards/margins": 0.901577353477478,
      "rewards/rejected": -1.2850050926208496,
      "step": 215
    },
    {
      "epoch": 1.3114754098360657,
      "grad_norm": 27.4957218170166,
      "learning_rate": 3.3709993123162106e-06,
      "log_odds_chosen": 1.301114797592163,
      "log_odds_ratio": -0.34698307514190674,
      "logits/chosen": 377.33697509765625,
      "logits/rejected": 371.03192138671875,
      "logps/chosen": -0.8284207582473755,
      "logps/rejected": -1.6063474416732788,
      "loss": 1.2242,
      "nll_loss": 1.0992552042007446,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.41421037912368774,
      "rewards/margins": 0.3889634311199188,
      "rewards/rejected": -0.8031737208366394,
      "step": 220
    },
    {
      "epoch": 1.3412816691505216,
      "grad_norm": 27.05774688720703,
      "learning_rate": 3.3333333333333333e-06,
      "log_odds_chosen": 1.0342134237289429,
      "log_odds_ratio": -0.3879423439502716,
      "logits/chosen": 373.6796875,
      "logits/rejected": 366.01123046875,
      "logps/chosen": -0.9923228025436401,
      "logps/rejected": -1.7060105800628662,
      "loss": 1.2664,
      "nll_loss": 1.197096586227417,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.49616140127182007,
      "rewards/margins": 0.3568439185619354,
      "rewards/rejected": -0.8530052900314331,
      "step": 225
    },
    {
      "epoch": 1.3710879284649775,
      "grad_norm": 32.180084228515625,
      "learning_rate": 3.296902366978936e-06,
      "log_odds_chosen": 1.9371551275253296,
      "log_odds_ratio": -0.265031635761261,
      "logits/chosen": 354.0810546875,
      "logits/rejected": 372.2239990234375,
      "logps/chosen": -0.8565861582756042,
      "logps/rejected": -2.277017116546631,
      "loss": 1.2422,
      "nll_loss": 0.9957939982414246,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.4282930791378021,
      "rewards/margins": 0.7102155685424805,
      "rewards/rejected": -1.1385085582733154,
      "step": 230
    },
    {
      "epoch": 1.4008941877794336,
      "grad_norm": 20.605178833007812,
      "learning_rate": 3.2616403652672114e-06,
      "log_odds_chosen": 1.8511009216308594,
      "log_odds_ratio": -0.2493407279253006,
      "logits/chosen": 376.54779052734375,
      "logits/rejected": 390.1648254394531,
      "logps/chosen": -0.7664806246757507,
      "logps/rejected": -2.081376075744629,
      "loss": 1.1997,
      "nll_loss": 1.0387040376663208,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.38324031233787537,
      "rewards/margins": 0.6574475765228271,
      "rewards/rejected": -1.0406880378723145,
      "step": 235
    },
    {
      "epoch": 1.4307004470938898,
      "grad_norm": 24.71708106994629,
      "learning_rate": 3.2274861218395142e-06,
      "log_odds_chosen": 1.3407671451568604,
      "log_odds_ratio": -0.33692649006843567,
      "logits/chosen": 401.45428466796875,
      "logits/rejected": 407.09161376953125,
      "logps/chosen": -0.9054458737373352,
      "logps/rejected": -1.8345670700073242,
      "loss": 1.3273,
      "nll_loss": 1.143675446510315,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.4527229368686676,
      "rewards/margins": 0.46456044912338257,
      "rewards/rejected": -0.9172835350036621,
      "step": 240
    },
    {
      "epoch": 1.4605067064083457,
      "grad_norm": 37.8891487121582,
      "learning_rate": 3.1943828249997e-06,
      "log_odds_chosen": 1.8552640676498413,
      "log_odds_ratio": -0.272559255361557,
      "logits/chosen": 400.0021667480469,
      "logits/rejected": 387.4122009277344,
      "logps/chosen": -0.7997323274612427,
      "logps/rejected": -2.1247220039367676,
      "loss": 1.3455,
      "nll_loss": 1.303070306777954,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.39986616373062134,
      "rewards/margins": 0.6624947786331177,
      "rewards/rejected": -1.0623610019683838,
      "step": 245
    },
    {
      "epoch": 1.4903129657228018,
      "grad_norm": 35.78718948364258,
      "learning_rate": 3.1622776601683796e-06,
      "log_odds_chosen": 1.5998339653015137,
      "log_odds_ratio": -0.3105994760990143,
      "logits/chosen": 369.63177490234375,
      "logits/rejected": 376.02960205078125,
      "logps/chosen": -0.7897557020187378,
      "logps/rejected": -1.8593695163726807,
      "loss": 1.2365,
      "nll_loss": 1.0359935760498047,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3948778510093689,
      "rewards/margins": 0.5348068475723267,
      "rewards/rejected": -0.9296847581863403,
      "step": 250
    },
    {
      "epoch": 1.520119225037258,
      "grad_norm": 25.08989143371582,
      "learning_rate": 3.131121455425748e-06,
      "log_odds_chosen": 1.8597294092178345,
      "log_odds_ratio": -0.21304909884929657,
      "logits/chosen": 384.16290283203125,
      "logits/rejected": 387.0345153808594,
      "logps/chosen": -0.7654696702957153,
      "logps/rejected": -2.019379138946533,
      "loss": 1.2499,
      "nll_loss": 1.030097246170044,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.38273483514785767,
      "rewards/margins": 0.6269546747207642,
      "rewards/rejected": -1.0096895694732666,
      "step": 255
    },
    {
      "epoch": 1.5499254843517138,
      "grad_norm": 21.55389976501465,
      "learning_rate": 3.1008683647302113e-06,
      "log_odds_chosen": 1.4927784204483032,
      "log_odds_ratio": -0.29883819818496704,
      "logits/chosen": 362.99444580078125,
      "logits/rejected": 404.9141845703125,
      "logps/chosen": -0.8609813451766968,
      "logps/rejected": -1.9159471988677979,
      "loss": 1.2228,
      "nll_loss": 1.09743070602417,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.4304906725883484,
      "rewards/margins": 0.5274828672409058,
      "rewards/rejected": -0.9579735994338989,
      "step": 260
    },
    {
      "epoch": 1.5797317436661698,
      "grad_norm": 19.87468910217285,
      "learning_rate": 3.0714755841697565e-06,
      "log_odds_chosen": 1.570449709892273,
      "log_odds_ratio": -0.3343026041984558,
      "logits/chosen": 374.11920166015625,
      "logits/rejected": 396.1978454589844,
      "logps/chosen": -0.8117960691452026,
      "logps/rejected": -1.8302360773086548,
      "loss": 1.3153,
      "nll_loss": 1.1015130281448364,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4058980345726013,
      "rewards/margins": 0.5092200040817261,
      "rewards/rejected": -0.9151180386543274,
      "step": 265
    },
    {
      "epoch": 1.6095380029806259,
      "grad_norm": 20.458053588867188,
      "learning_rate": 3.0429030972509227e-06,
      "log_odds_chosen": 1.468200922012329,
      "log_odds_ratio": -0.3185100853443146,
      "logits/chosen": 360.3885192871094,
      "logits/rejected": 371.0684509277344,
      "logps/chosen": -0.948271632194519,
      "logps/rejected": -1.9939063787460327,
      "loss": 1.2987,
      "nll_loss": 1.2175120115280151,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.4741358160972595,
      "rewards/margins": 0.5228174328804016,
      "rewards/rejected": -0.9969531893730164,
      "step": 270
    },
    {
      "epoch": 1.639344262295082,
      "grad_norm": 21.885934829711914,
      "learning_rate": 3.0151134457776365e-06,
      "log_odds_chosen": 1.2957884073257446,
      "log_odds_ratio": -0.34294062852859497,
      "logits/chosen": 351.11566162109375,
      "logits/rejected": 341.6363830566406,
      "logps/chosen": -0.816031277179718,
      "logps/rejected": -1.6235980987548828,
      "loss": 1.3141,
      "nll_loss": 1.0874848365783691,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.408015638589859,
      "rewards/margins": 0.40378338098526,
      "rewards/rejected": -0.8117990493774414,
      "step": 275
    },
    {
      "epoch": 1.669150521609538,
      "grad_norm": 32.71710205078125,
      "learning_rate": 2.988071523335984e-06,
      "log_odds_chosen": 1.3881560564041138,
      "log_odds_ratio": -0.44494813680648804,
      "logits/chosen": 399.6920471191406,
      "logits/rejected": 387.6843566894531,
      "logps/chosen": -0.9040805697441101,
      "logps/rejected": -1.8093194961547852,
      "loss": 1.3021,
      "nll_loss": 1.1397716999053955,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.45204028487205505,
      "rewards/margins": 0.4526194930076599,
      "rewards/rejected": -0.9046597480773926,
      "step": 280
    },
    {
      "epoch": 1.698956780923994,
      "grad_norm": 31.200809478759766,
      "learning_rate": 2.961744388795462e-06,
      "log_odds_chosen": 1.7313816547393799,
      "log_odds_ratio": -0.28469711542129517,
      "logits/chosen": 359.5746154785156,
      "logits/rejected": 362.81494140625,
      "logps/chosen": -0.6776639223098755,
      "logps/rejected": -1.7751470804214478,
      "loss": 1.2089,
      "nll_loss": 1.0214917659759521,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.33883196115493774,
      "rewards/margins": 0.5487415790557861,
      "rewards/rejected": -0.8875735402107239,
      "step": 285
    },
    {
      "epoch": 1.7287630402384502,
      "grad_norm": 17.744766235351562,
      "learning_rate": 2.9361010975735177e-06,
      "log_odds_chosen": 1.5064774751663208,
      "log_odds_ratio": -0.3122694492340088,
      "logits/chosen": 376.41033935546875,
      "logits/rejected": 415.8057556152344,
      "logps/chosen": -0.8741256594657898,
      "logps/rejected": -1.9100143909454346,
      "loss": 1.259,
      "nll_loss": 1.0986948013305664,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.4370628297328949,
      "rewards/margins": 0.5179442167282104,
      "rewards/rejected": -0.9550071954727173,
      "step": 290
    },
    {
      "epoch": 1.758569299552906,
      "grad_norm": 22.985361099243164,
      "learning_rate": 2.9111125486979104e-06,
      "log_odds_chosen": 1.4937174320220947,
      "log_odds_ratio": -0.3182430565357208,
      "logits/chosen": 353.3531188964844,
      "logits/rejected": 398.23382568359375,
      "logps/chosen": -0.8985480070114136,
      "logps/rejected": -1.8789575099945068,
      "loss": 1.2814,
      "nll_loss": 1.1132934093475342,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4492740035057068,
      "rewards/margins": 0.4902048110961914,
      "rewards/rejected": -0.9394787549972534,
      "step": 295
    },
    {
      "epoch": 1.788375558867362,
      "grad_norm": 25.093547821044922,
      "learning_rate": 2.8867513459481293e-06,
      "log_odds_chosen": 1.7519241571426392,
      "log_odds_ratio": -0.23565292358398438,
      "logits/chosen": 396.87640380859375,
      "logits/rejected": 374.09942626953125,
      "logps/chosen": -0.6939581632614136,
      "logps/rejected": -1.8357816934585571,
      "loss": 1.201,
      "nll_loss": 0.9846609234809875,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3469790816307068,
      "rewards/margins": 0.5709116458892822,
      "rewards/rejected": -0.9178908467292786,
      "step": 300
    },
    {
      "epoch": 1.8181818181818183,
      "grad_norm": 22.51293182373047,
      "learning_rate": 2.862991671569341e-06,
      "log_odds_chosen": 1.3752100467681885,
      "log_odds_ratio": -0.39724329113960266,
      "logits/chosen": 392.15093994140625,
      "logits/rejected": 400.69677734375,
      "logps/chosen": -0.9685919880867004,
      "logps/rejected": -2.020549774169922,
      "loss": 1.2557,
      "nll_loss": 1.2231824398040771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4842959940433502,
      "rewards/margins": 0.5259788036346436,
      "rewards/rejected": -1.010274887084961,
      "step": 305
    },
    {
      "epoch": 1.8479880774962743,
      "grad_norm": 18.64189910888672,
      "learning_rate": 2.839809171235324e-06,
      "log_odds_chosen": 1.9540401697158813,
      "log_odds_ratio": -0.28014156222343445,
      "logits/chosen": 374.20074462890625,
      "logits/rejected": 382.771240234375,
      "logps/chosen": -0.823870837688446,
      "logps/rejected": -2.250458002090454,
      "loss": 1.3098,
      "nll_loss": 1.1509441137313843,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.411935418844223,
      "rewards/margins": 0.7132935523986816,
      "rewards/rejected": -1.125229001045227,
      "step": 310
    },
    {
      "epoch": 1.8777943368107302,
      "grad_norm": 20.656198501586914,
      "learning_rate": 2.817180849095055e-06,
      "log_odds_chosen": 0.9874370694160461,
      "log_odds_ratio": -0.43443575501441956,
      "logits/chosen": 348.67279052734375,
      "logits/rejected": 367.7172546386719,
      "logps/chosen": -1.063990831375122,
      "logps/rejected": -1.7773240804672241,
      "loss": 1.3159,
      "nll_loss": 1.2891852855682373,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.531995415687561,
      "rewards/margins": 0.3566665053367615,
      "rewards/rejected": -0.8886620402336121,
      "step": 315
    },
    {
      "epoch": 1.9076005961251863,
      "grad_norm": 22.823558807373047,
      "learning_rate": 2.7950849718747376e-06,
      "log_odds_chosen": 1.9457191228866577,
      "log_odds_ratio": -0.23255281150341034,
      "logits/chosen": 369.242431640625,
      "logits/rejected": 389.74346923828125,
      "logps/chosen": -0.7030061483383179,
      "logps/rejected": -2.0039026737213135,
      "loss": 1.1894,
      "nll_loss": 1.0204839706420898,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.35150307416915894,
      "rewards/margins": 0.650448203086853,
      "rewards/rejected": -1.0019513368606567,
      "step": 320
    },
    {
      "epoch": 1.9374068554396424,
      "grad_norm": 27.562149047851562,
      "learning_rate": 2.773500981126146e-06,
      "log_odds_chosen": 1.998150110244751,
      "log_odds_ratio": -0.24417133629322052,
      "logits/chosen": 365.95684814453125,
      "logits/rejected": 397.9959411621094,
      "logps/chosen": -0.7909399271011353,
      "logps/rejected": -2.2285313606262207,
      "loss": 1.179,
      "nll_loss": 1.0059034824371338,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.3954699635505676,
      "rewards/margins": 0.7187958359718323,
      "rewards/rejected": -1.1142656803131104,
      "step": 325
    },
    {
      "epoch": 1.9672131147540983,
      "grad_norm": 27.540767669677734,
      "learning_rate": 2.752409412815902e-06,
      "log_odds_chosen": 1.38077712059021,
      "log_odds_ratio": -0.3091193735599518,
      "logits/chosen": 361.24969482421875,
      "logits/rejected": 369.6480407714844,
      "logps/chosen": -0.8252070546150208,
      "logps/rejected": -1.7504304647445679,
      "loss": 1.2534,
      "nll_loss": 0.9596601724624634,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4126035273075104,
      "rewards/margins": 0.46261176466941833,
      "rewards/rejected": -0.8752152323722839,
      "step": 330
    },
    {
      "epoch": 1.9970193740685542,
      "grad_norm": 35.79298782348633,
      "learning_rate": 2.7317918235407652e-06,
      "log_odds_chosen": 1.0996582508087158,
      "log_odds_ratio": -0.4153285622596741,
      "logits/chosen": 389.44366455078125,
      "logits/rejected": 379.0640563964844,
      "logps/chosen": -1.0918314456939697,
      "logps/rejected": -1.8735624551773071,
      "loss": 1.3092,
      "nll_loss": 1.2989994287490845,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.5459157228469849,
      "rewards/margins": 0.3908655047416687,
      "rewards/rejected": -0.9367812275886536,
      "step": 335
    },
    {
      "epoch": 1.9970193740685542,
      "eval_log_odds_chosen": 0.32236072421073914,
      "eval_log_odds_ratio": -0.6637296676635742,
      "eval_logits/chosen": 309.3900146484375,
      "eval_logits/rejected": 279.0658874511719,
      "eval_logps/chosen": -1.040497064590454,
      "eval_logps/rejected": -1.2617474794387817,
      "eval_loss": 1.7559822797775269,
      "eval_nll_loss": 1.4054137468338013,
      "eval_rewards/accuracies": 0.5323740839958191,
      "eval_rewards/chosen": -0.520248532295227,
      "eval_rewards/margins": 0.11062520742416382,
      "eval_rewards/rejected": -0.6308737397193909,
      "eval_runtime": 351.6864,
      "eval_samples_per_second": 1.572,
      "eval_steps_per_second": 0.395,
      "step": 335
    },
    {
      "epoch": 2.0268256333830106,
      "grad_norm": 22.68450927734375,
      "learning_rate": 2.711630722733202e-06,
      "log_odds_chosen": 3.2875800132751465,
      "log_odds_ratio": -0.10148422420024872,
      "logits/chosen": 390.20111083984375,
      "logits/rejected": 362.7218017578125,
      "logps/chosen": -0.4825616478919983,
      "logps/rejected": -2.6374728679656982,
      "loss": 0.7684,
      "nll_loss": 0.8495180010795593,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.24128082394599915,
      "rewards/margins": 1.0774555206298828,
      "rewards/rejected": -1.3187364339828491,
      "step": 340
    },
    {
      "epoch": 2.0566318926974665,
      "grad_norm": 12.678057670593262,
      "learning_rate": 2.691909510290828e-06,
      "log_odds_chosen": 3.9607760906219482,
      "log_odds_ratio": -0.06652864068746567,
      "logits/chosen": 340.1753234863281,
      "logits/rejected": 337.1167907714844,
      "logps/chosen": -0.44624167680740356,
      "logps/rejected": -3.1436822414398193,
      "loss": 0.6642,
      "nll_loss": 0.6589840650558472,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.22312083840370178,
      "rewards/margins": 1.3487205505371094,
      "rewards/rejected": -1.5718411207199097,
      "step": 345
    },
    {
      "epoch": 2.0864381520119224,
      "grad_norm": 18.899171829223633,
      "learning_rate": 2.6726124191242444e-06,
      "log_odds_chosen": 3.980642318725586,
      "log_odds_ratio": -0.04676336795091629,
      "logits/chosen": 340.1651916503906,
      "logits/rejected": 373.43426513671875,
      "logps/chosen": -0.43081822991371155,
      "logps/rejected": -3.3248093128204346,
      "loss": 0.6728,
      "nll_loss": 0.6396235227584839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21540911495685577,
      "rewards/margins": 1.446995496749878,
      "rewards/rejected": -1.6624046564102173,
      "step": 350
    },
    {
      "epoch": 2.1162444113263787,
      "grad_norm": 15.860565185546875,
      "learning_rate": 2.6537244621713765e-06,
      "log_odds_chosen": 3.6839728355407715,
      "log_odds_ratio": -0.07649212330579758,
      "logits/chosen": 345.269287109375,
      "logits/rejected": 359.37823486328125,
      "logps/chosen": -0.45684829354286194,
      "logps/rejected": -3.046020984649658,
      "loss": 0.666,
      "nll_loss": 0.7428454160690308,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.22842414677143097,
      "rewards/margins": 1.294586420059204,
      "rewards/rejected": -1.523010492324829,
      "step": 355
    },
    {
      "epoch": 2.1460506706408347,
      "grad_norm": 13.459833145141602,
      "learning_rate": 2.6352313834736496e-06,
      "log_odds_chosen": 4.002976417541504,
      "log_odds_ratio": -0.04872741550207138,
      "logits/chosen": 340.22161865234375,
      "logits/rejected": 384.5272521972656,
      "logps/chosen": -0.433218777179718,
      "logps/rejected": -3.1645781993865967,
      "loss": 0.6602,
      "nll_loss": 0.6463421583175659,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.216609388589859,
      "rewards/margins": 1.3656798601150513,
      "rewards/rejected": -1.5822890996932983,
      "step": 360
    },
    {
      "epoch": 2.1758569299552906,
      "grad_norm": 14.115008354187012,
      "learning_rate": 2.6171196129510684e-06,
      "log_odds_chosen": 3.177072763442993,
      "log_odds_ratio": -0.0845123901963234,
      "logits/chosen": 326.1507568359375,
      "logits/rejected": 306.5514831542969,
      "logps/chosen": -0.41812819242477417,
      "logps/rejected": -2.4115400314331055,
      "loss": 0.6679,
      "nll_loss": 0.6390963196754456,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.20906409621238708,
      "rewards/margins": 0.9967058300971985,
      "rewards/rejected": -1.2057700157165527,
      "step": 365
    },
    {
      "epoch": 2.2056631892697465,
      "grad_norm": 15.614830017089844,
      "learning_rate": 2.599376224550182e-06,
      "log_odds_chosen": 3.3914806842803955,
      "log_odds_ratio": -0.08955641835927963,
      "logits/chosen": 287.53955078125,
      "logits/rejected": 306.90667724609375,
      "logps/chosen": -0.44138556718826294,
      "logps/rejected": -2.749290943145752,
      "loss": 0.6969,
      "nll_loss": 0.7107062935829163,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.22069278359413147,
      "rewards/margins": 1.153952717781067,
      "rewards/rejected": -1.374645471572876,
      "step": 370
    },
    {
      "epoch": 2.235469448584203,
      "grad_norm": 15.579936027526855,
      "learning_rate": 2.5819888974716113e-06,
      "log_odds_chosen": 3.3506603240966797,
      "log_odds_ratio": -0.09304364025592804,
      "logits/chosen": 339.9989929199219,
      "logits/rejected": 357.97528076171875,
      "logps/chosen": -0.5137821435928345,
      "logps/rejected": -2.7970046997070312,
      "loss": 0.6856,
      "nll_loss": 0.7090636491775513,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.25689107179641724,
      "rewards/margins": 1.1416113376617432,
      "rewards/rejected": -1.3985023498535156,
      "step": 375
    },
    {
      "epoch": 2.2652757078986587,
      "grad_norm": 17.770183563232422,
      "learning_rate": 2.564945880212886e-06,
      "log_odds_chosen": 3.6279399394989014,
      "log_odds_ratio": -0.04611171409487724,
      "logits/chosen": 343.6501159667969,
      "logits/rejected": 322.5646057128906,
      "logps/chosen": -0.35303473472595215,
      "logps/rejected": -2.671903610229492,
      "loss": 0.6689,
      "nll_loss": 0.6217206120491028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17651736736297607,
      "rewards/margins": 1.1594343185424805,
      "rewards/rejected": -1.335951805114746,
      "step": 380
    },
    {
      "epoch": 2.2950819672131146,
      "grad_norm": 12.41324520111084,
      "learning_rate": 2.5482359571881276e-06,
      "log_odds_chosen": 3.9699618816375732,
      "log_odds_ratio": -0.04786146804690361,
      "logits/chosen": 341.5869445800781,
      "logits/rejected": 333.04608154296875,
      "logps/chosen": -0.3405749499797821,
      "logps/rejected": -2.8487496376037598,
      "loss": 0.6364,
      "nll_loss": 0.5744936466217041,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.17028747498989105,
      "rewards/margins": 1.254087209701538,
      "rewards/rejected": -1.4243748188018799,
      "step": 385
    },
    {
      "epoch": 2.3248882265275705,
      "grad_norm": 14.415823936462402,
      "learning_rate": 2.5318484177091667e-06,
      "log_odds_chosen": 3.405970335006714,
      "log_odds_ratio": -0.05260590463876724,
      "logits/chosen": 360.901611328125,
      "logits/rejected": 378.63616943359375,
      "logps/chosen": -0.5018097162246704,
      "logps/rejected": -2.8674509525299072,
      "loss": 0.7074,
      "nll_loss": 0.6968181729316711,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2509048581123352,
      "rewards/margins": 1.1828205585479736,
      "rewards/rejected": -1.4337254762649536,
      "step": 390
    },
    {
      "epoch": 2.354694485842027,
      "grad_norm": 11.383499145507812,
      "learning_rate": 2.515773027133138e-06,
      "log_odds_chosen": 3.9227728843688965,
      "log_odds_ratio": -0.05491284653544426,
      "logits/chosen": 349.3078308105469,
      "logits/rejected": 332.0811462402344,
      "logps/chosen": -0.32619425654411316,
      "logps/rejected": -2.7826080322265625,
      "loss": 0.6264,
      "nll_loss": 0.5774807929992676,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.16309712827205658,
      "rewards/margins": 1.2282068729400635,
      "rewards/rejected": -1.3913040161132812,
      "step": 395
    },
    {
      "epoch": 2.384500745156483,
      "grad_norm": 15.452503204345703,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 3.7418270111083984,
      "log_odds_ratio": -0.08289723098278046,
      "logits/chosen": 353.244873046875,
      "logits/rejected": 371.33941650390625,
      "logps/chosen": -0.4247261583805084,
      "logps/rejected": -3.0048041343688965,
      "loss": 0.6678,
      "nll_loss": 0.5866397619247437,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.2123630791902542,
      "rewards/margins": 1.2900387048721313,
      "rewards/rejected": -1.5024020671844482,
      "step": 400
    },
    {
      "epoch": 2.4143070044709387,
      "grad_norm": 14.744851112365723,
      "learning_rate": 2.484519974999767e-06,
      "log_odds_chosen": 4.066527366638184,
      "log_odds_ratio": -0.07095837593078613,
      "logits/chosen": 405.4268493652344,
      "logits/rejected": 363.4831237792969,
      "logps/chosen": -0.4019082188606262,
      "logps/rejected": -3.0571980476379395,
      "loss": 0.6775,
      "nll_loss": 0.6186262965202332,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.2009541094303131,
      "rewards/margins": 1.3276448249816895,
      "rewards/rejected": -1.5285990238189697,
      "step": 405
    },
    {
      "epoch": 2.444113263785395,
      "grad_norm": 15.641300201416016,
      "learning_rate": 2.4693239916239746e-06,
      "log_odds_chosen": 3.6895904541015625,
      "log_odds_ratio": -0.08013327419757843,
      "logits/chosen": 352.7126770019531,
      "logits/rejected": 362.0486145019531,
      "logps/chosen": -0.4512965679168701,
      "logps/rejected": -2.8881852626800537,
      "loss": 0.6612,
      "nll_loss": 0.6566362380981445,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.22564828395843506,
      "rewards/margins": 1.2184442281723022,
      "rewards/rejected": -1.4440926313400269,
      "step": 410
    },
    {
      "epoch": 2.473919523099851,
      "grad_norm": 13.112720489501953,
      "learning_rate": 2.4544034683690802e-06,
      "log_odds_chosen": 4.032966613769531,
      "log_odds_ratio": -0.03849425166845322,
      "logits/chosen": 352.24176025390625,
      "logits/rejected": 376.4973449707031,
      "logps/chosen": -0.38949456810951233,
      "logps/rejected": -3.1059412956237793,
      "loss": 0.6593,
      "nll_loss": 0.5457557439804077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19474728405475616,
      "rewards/margins": 1.35822331905365,
      "rewards/rejected": -1.5529706478118896,
      "step": 415
    },
    {
      "epoch": 2.503725782414307,
      "grad_norm": 13.079460144042969,
      "learning_rate": 2.4397501823713327e-06,
      "log_odds_chosen": 3.7700283527374268,
      "log_odds_ratio": -0.06273230165243149,
      "logits/chosen": 351.4734802246094,
      "logits/rejected": 317.71209716796875,
      "logps/chosen": -0.3789517283439636,
      "logps/rejected": -2.9251832962036133,
      "loss": 0.6409,
      "nll_loss": 0.7009339928627014,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.1894758641719818,
      "rewards/margins": 1.273115634918213,
      "rewards/rejected": -1.4625916481018066,
      "step": 420
    },
    {
      "epoch": 2.533532041728763,
      "grad_norm": 20.27842903137207,
      "learning_rate": 2.4253562503633297e-06,
      "log_odds_chosen": 4.24957799911499,
      "log_odds_ratio": -0.032464753836393356,
      "logits/chosen": 348.3805847167969,
      "logits/rejected": 337.718994140625,
      "logps/chosen": -0.3897190988063812,
      "logps/rejected": -3.382542848587036,
      "loss": 0.6474,
      "nll_loss": 0.6459823250770569,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1948595494031906,
      "rewards/margins": 1.4964120388031006,
      "rewards/rejected": -1.691271424293518,
      "step": 425
    },
    {
      "epoch": 2.563338301043219,
      "grad_norm": 14.640429496765137,
      "learning_rate": 2.411214110852061e-06,
      "log_odds_chosen": 4.611870765686035,
      "log_odds_ratio": -0.03490515798330307,
      "logits/chosen": 347.74298095703125,
      "logits/rejected": 351.59912109375,
      "logps/chosen": -0.3038939833641052,
      "logps/rejected": -3.4484775066375732,
      "loss": 0.6441,
      "nll_loss": 0.5448944568634033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1519469916820526,
      "rewards/margins": 1.572291612625122,
      "rewards/rejected": -1.7242387533187866,
      "step": 430
    },
    {
      "epoch": 2.593144560357675,
      "grad_norm": 15.337629318237305,
      "learning_rate": 2.3973165074269213e-06,
      "log_odds_chosen": 3.7105846405029297,
      "log_odds_ratio": -0.04743606597185135,
      "logits/chosen": 360.7786560058594,
      "logits/rejected": 318.19854736328125,
      "logps/chosen": -0.36348485946655273,
      "logps/rejected": -2.7734932899475098,
      "loss": 0.6432,
      "nll_loss": 0.5634604692459106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18174242973327637,
      "rewards/margins": 1.205004096031189,
      "rewards/rejected": -1.3867466449737549,
      "step": 435
    },
    {
      "epoch": 2.6229508196721314,
      "grad_norm": 13.271897315979004,
      "learning_rate": 2.3836564731139807e-06,
      "log_odds_chosen": 3.826213836669922,
      "log_odds_ratio": -0.04420660808682442,
      "logits/chosen": 342.50848388671875,
      "logits/rejected": 348.86456298828125,
      "logps/chosen": -0.30872535705566406,
      "logps/rejected": -2.6104750633239746,
      "loss": 0.666,
      "nll_loss": 0.6212347745895386,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15436267852783203,
      "rewards/margins": 1.1508748531341553,
      "rewards/rejected": -1.3052375316619873,
      "step": 440
    },
    {
      "epoch": 2.6527570789865873,
      "grad_norm": 17.46467399597168,
      "learning_rate": 2.3702273156998867e-06,
      "log_odds_chosen": 4.261384963989258,
      "log_odds_ratio": -0.03544224798679352,
      "logits/chosen": 323.58251953125,
      "logits/rejected": 354.87847900390625,
      "logps/chosen": -0.4285755753517151,
      "logps/rejected": -3.4066290855407715,
      "loss": 0.6866,
      "nll_loss": 0.6935943961143494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21428778767585754,
      "rewards/margins": 1.4890267848968506,
      "rewards/rejected": -1.7033145427703857,
      "step": 445
    },
    {
      "epoch": 2.682563338301043,
      "grad_norm": 12.280491828918457,
      "learning_rate": 2.357022603955159e-06,
      "log_odds_chosen": 3.510754108428955,
      "log_odds_ratio": -0.05934901908040047,
      "logits/chosen": 356.21588134765625,
      "logits/rejected": 349.2935485839844,
      "logps/chosen": -0.4579055905342102,
      "logps/rejected": -2.802351474761963,
      "loss": 0.6947,
      "nll_loss": 0.6444963216781616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2289527952671051,
      "rewards/margins": 1.1722228527069092,
      "rewards/rejected": -1.4011757373809814,
      "step": 450
    },
    {
      "epoch": 2.712369597615499,
      "grad_norm": 14.349300384521484,
      "learning_rate": 2.3440361546924774e-06,
      "log_odds_chosen": 3.9128804206848145,
      "log_odds_ratio": -0.05345267057418823,
      "logits/chosen": 379.7029724121094,
      "logits/rejected": 353.4925842285156,
      "logps/chosen": -0.4496696889400482,
      "logps/rejected": -3.064148426055908,
      "loss": 0.7279,
      "nll_loss": 0.6569377779960632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2248348444700241,
      "rewards/margins": 1.307239294052124,
      "rewards/rejected": -1.532074213027954,
      "step": 455
    },
    {
      "epoch": 2.742175856929955,
      "grad_norm": 13.55716609954834,
      "learning_rate": 2.3312620206007847e-06,
      "log_odds_chosen": 4.291790962219238,
      "log_odds_ratio": -0.044444940984249115,
      "logits/chosen": 374.377197265625,
      "logits/rejected": 386.43414306640625,
      "logps/chosen": -0.3812227249145508,
      "logps/rejected": -3.4395527839660645,
      "loss": 0.6847,
      "nll_loss": 0.7294474840164185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1906113624572754,
      "rewards/margins": 1.5291650295257568,
      "rewards/rejected": -1.7197763919830322,
      "step": 460
    },
    {
      "epoch": 2.7719821162444114,
      "grad_norm": 13.988215446472168,
      "learning_rate": 2.3186944788008413e-06,
      "log_odds_chosen": 4.281071186065674,
      "log_odds_ratio": -0.04669572785496712,
      "logits/chosen": 359.35101318359375,
      "logits/rejected": 352.8810729980469,
      "logps/chosen": -0.30659520626068115,
      "logps/rejected": -3.1890709400177,
      "loss": 0.6956,
      "nll_loss": 0.6471156477928162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15329760313034058,
      "rewards/margins": 1.4412376880645752,
      "rewards/rejected": -1.59453547000885,
      "step": 465
    },
    {
      "epoch": 2.8017883755588673,
      "grad_norm": 16.967275619506836,
      "learning_rate": 2.3063280200722128e-06,
      "log_odds_chosen": 3.476628065109253,
      "log_odds_ratio": -0.07849601656198502,
      "logits/chosen": 363.3095703125,
      "logits/rejected": 323.81842041015625,
      "logps/chosen": -0.43367767333984375,
      "logps/rejected": -2.7612318992614746,
      "loss": 0.6725,
      "nll_loss": 0.6623119711875916,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.21683883666992188,
      "rewards/margins": 1.1637769937515259,
      "rewards/rejected": -1.3806159496307373,
      "step": 470
    },
    {
      "epoch": 2.8315946348733236,
      "grad_norm": 19.491451263427734,
      "learning_rate": 2.2941573387056174e-06,
      "log_odds_chosen": 3.7874722480773926,
      "log_odds_ratio": -0.04359927773475647,
      "logits/chosen": 332.71844482421875,
      "logits/rejected": 350.5658264160156,
      "logps/chosen": -0.39931654930114746,
      "logps/rejected": -2.8959929943084717,
      "loss": 0.6421,
      "nll_loss": 0.5615902543067932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19965827465057373,
      "rewards/margins": 1.248338222503662,
      "rewards/rejected": -1.4479964971542358,
      "step": 475
    },
    {
      "epoch": 2.8614008941877795,
      "grad_norm": 18.62973976135254,
      "learning_rate": 2.2821773229381924e-06,
      "log_odds_chosen": 4.226737976074219,
      "log_odds_ratio": -0.04461986571550369,
      "logits/chosen": 344.9051208496094,
      "logits/rejected": 380.61822509765625,
      "logps/chosen": -0.3830781877040863,
      "logps/rejected": -3.3659965991973877,
      "loss": 0.6189,
      "nll_loss": 0.5661858320236206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19153909385204315,
      "rewards/margins": 1.4914592504501343,
      "rewards/rejected": -1.6829982995986938,
      "step": 480
    },
    {
      "epoch": 2.8912071535022354,
      "grad_norm": 14.639021873474121,
      "learning_rate": 2.270383045932499e-06,
      "log_odds_chosen": 3.5930495262145996,
      "log_odds_ratio": -0.06206268072128296,
      "logits/chosen": 341.22650146484375,
      "logits/rejected": 361.0518493652344,
      "logps/chosen": -0.48062896728515625,
      "logps/rejected": -2.968130111694336,
      "loss": 0.6645,
      "nll_loss": 0.6335129737854004,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.24031448364257812,
      "rewards/margins": 1.2437504529953003,
      "rewards/rejected": -1.484065055847168,
      "step": 485
    },
    {
      "epoch": 2.9210134128166914,
      "grad_norm": 13.041991233825684,
      "learning_rate": 2.2587697572631284e-06,
      "log_odds_chosen": 3.418093204498291,
      "log_odds_ratio": -0.06181482598185539,
      "logits/chosen": 360.23419189453125,
      "logits/rejected": 314.44183349609375,
      "logps/chosen": -0.4381326138973236,
      "logps/rejected": -2.6530632972717285,
      "loss": 0.7257,
      "nll_loss": 0.6311744451522827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2190663069486618,
      "rewards/margins": 1.107465386390686,
      "rewards/rejected": -1.3265316486358643,
      "step": 490
    },
    {
      "epoch": 2.9508196721311473,
      "grad_norm": 13.917710304260254,
      "learning_rate": 2.2473328748774737e-06,
      "log_odds_chosen": 3.9481112957000732,
      "log_odds_ratio": -0.07351940125226974,
      "logits/chosen": 354.31781005859375,
      "logits/rejected": 374.8128356933594,
      "logps/chosen": -0.5134202837944031,
      "logps/rejected": -3.2178916931152344,
      "loss": 0.6495,
      "nll_loss": 0.6444075703620911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.25671014189720154,
      "rewards/margins": 1.3522356748580933,
      "rewards/rejected": -1.6089458465576172,
      "step": 495
    },
    {
      "epoch": 2.9806259314456036,
      "grad_norm": 13.279515266418457,
      "learning_rate": 2.23606797749979e-06,
      "log_odds_chosen": 3.631924867630005,
      "log_odds_ratio": -0.05388236045837402,
      "logits/chosen": 363.3906555175781,
      "logits/rejected": 350.91546630859375,
      "logps/chosen": -0.3526715636253357,
      "logps/rejected": -2.695366382598877,
      "loss": 0.6827,
      "nll_loss": 0.5351300835609436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17633578181266785,
      "rewards/margins": 1.1713473796844482,
      "rewards/rejected": -1.3476831912994385,
      "step": 500
    },
    {
      "epoch": 2.9865871833084947,
      "eval_log_odds_chosen": 0.42297032475471497,
      "eval_log_odds_ratio": -0.6477048993110657,
      "eval_logits/chosen": 282.00848388671875,
      "eval_logits/rejected": 247.6262664794922,
      "eval_logps/chosen": -1.2038370370864868,
      "eval_logps/rejected": -1.5014722347259521,
      "eval_loss": 1.9079890251159668,
      "eval_nll_loss": 1.5545235872268677,
      "eval_rewards/accuracies": 0.6258992552757263,
      "eval_rewards/chosen": -0.6019185185432434,
      "eval_rewards/margins": 0.14881758391857147,
      "eval_rewards/rejected": -0.7507361173629761,
      "eval_runtime": 351.9872,
      "eval_samples_per_second": 1.571,
      "eval_steps_per_second": 0.395,
      "step": 501
    },
    {
      "epoch": 2.9865871833084947,
      "step": 501,
      "total_flos": 0.0,
      "train_loss": 1.686337627812536,
      "train_runtime": 31593.308,
      "train_samples_per_second": 0.509,
      "train_steps_per_second": 0.016
    }
  ],
  "logging_steps": 5,
  "max_steps": 501,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}