{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9987943737441393,
  "eval_steps": 400,
  "global_step": 466,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010716677829872739,
      "grad_norm": 37.4633897932904,
      "learning_rate": 2.1276595744680853e-06,
      "logits/chosen": -2.397902250289917,
      "logits/rejected": -2.391808271408081,
      "logps/chosen": -0.5661157369613647,
      "logps/rejected": -0.5547963380813599,
      "loss": 1.5468,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5661157369613647,
      "rewards/margins": -0.011319375596940517,
      "rewards/rejected": -0.5547963380813599,
      "step": 5
    },
    {
      "epoch": 0.021433355659745478,
      "grad_norm": 18.848503699490514,
      "learning_rate": 4.255319148936171e-06,
      "logits/chosen": -2.374025344848633,
      "logits/rejected": -2.345165729522705,
      "logps/chosen": -0.5381986498832703,
      "logps/rejected": -0.5637714266777039,
      "loss": 1.5538,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.5381986498832703,
      "rewards/margins": 0.025572726503014565,
      "rewards/rejected": -0.5637714266777039,
      "step": 10
    },
    {
      "epoch": 0.032150033489618215,
      "grad_norm": 18.865749434279863,
      "learning_rate": 6.382978723404256e-06,
      "logits/chosen": -2.228337526321411,
      "logits/rejected": -2.2436676025390625,
      "logps/chosen": -0.6078538298606873,
      "logps/rejected": -0.6489665508270264,
      "loss": 1.5486,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.6078538298606873,
      "rewards/margins": 0.04111265391111374,
      "rewards/rejected": -0.6489665508270264,
      "step": 15
    },
    {
      "epoch": 0.042866711319490956,
      "grad_norm": 11.695416903513719,
      "learning_rate": 8.510638297872341e-06,
      "logits/chosen": -2.3822503089904785,
      "logits/rejected": -2.399451732635498,
      "logps/chosen": -0.6396132111549377,
      "logps/rejected": -0.6679492592811584,
      "loss": 1.5283,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.6396132111549377,
      "rewards/margins": 0.028336137533187866,
      "rewards/rejected": -0.6679492592811584,
      "step": 20
    },
    {
      "epoch": 0.0535833891493637,
      "grad_norm": 8.657694670203071,
      "learning_rate": 1.0638297872340426e-05,
      "logits/chosen": -2.6283469200134277,
      "logits/rejected": -2.6289353370666504,
      "logps/chosen": -0.6763823628425598,
      "logps/rejected": -0.7394057512283325,
      "loss": 1.5522,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.6763823628425598,
      "rewards/margins": 0.0630234032869339,
      "rewards/rejected": -0.7394057512283325,
      "step": 25
    },
    {
      "epoch": 0.06430006697923643,
      "grad_norm": 11.701639197469074,
      "learning_rate": 1.2765957446808513e-05,
      "logits/chosen": -2.7873454093933105,
      "logits/rejected": -2.815898895263672,
      "logps/chosen": -0.8007046580314636,
      "logps/rejected": -0.8376566767692566,
      "loss": 1.5826,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.8007046580314636,
      "rewards/margins": 0.03695206344127655,
      "rewards/rejected": -0.8376566767692566,
      "step": 30
    },
    {
      "epoch": 0.07501674480910918,
      "grad_norm": 85.64689375367887,
      "learning_rate": 1.4893617021276596e-05,
      "logits/chosen": -2.5493829250335693,
      "logits/rejected": -2.5297470092773438,
      "logps/chosen": -0.9185592532157898,
      "logps/rejected": -1.0066051483154297,
      "loss": 1.6494,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.9185592532157898,
      "rewards/margins": 0.08804582059383392,
      "rewards/rejected": -1.0066051483154297,
      "step": 35
    },
    {
      "epoch": 0.08573342263898191,
      "grad_norm": 12.618515943070657,
      "learning_rate": 1.7021276595744682e-05,
      "logits/chosen": -2.7925362586975098,
      "logits/rejected": -2.7926783561706543,
      "logps/chosen": -0.95704585313797,
      "logps/rejected": -0.9239652752876282,
      "loss": 1.6286,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.95704585313797,
      "rewards/margins": -0.033080630004405975,
      "rewards/rejected": -0.9239652752876282,
      "step": 40
    },
    {
      "epoch": 0.09645010046885466,
      "grad_norm": 7.437533737509709,
      "learning_rate": 1.914893617021277e-05,
      "logits/chosen": -2.668349027633667,
      "logits/rejected": -2.6576356887817383,
      "logps/chosen": -0.9402163624763489,
      "logps/rejected": -0.9749400019645691,
      "loss": 1.5995,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.9402163624763489,
      "rewards/margins": 0.034723617136478424,
      "rewards/rejected": -0.9749400019645691,
      "step": 45
    },
    {
      "epoch": 0.1071667782987274,
      "grad_norm": 8.099072281553982,
      "learning_rate": 1.999747031816659e-05,
      "logits/chosen": -2.796558380126953,
      "logits/rejected": -2.7897918224334717,
      "logps/chosen": -0.8397698402404785,
      "logps/rejected": -0.8385842442512512,
      "loss": 1.566,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.8397698402404785,
      "rewards/margins": -0.0011855497723445296,
      "rewards/rejected": -0.8385842442512512,
      "step": 50
    },
    {
      "epoch": 0.11788345612860013,
      "grad_norm": 13.021189847208735,
      "learning_rate": 1.9982015786040484e-05,
      "logits/chosen": -2.577963352203369,
      "logits/rejected": -2.5813965797424316,
      "logps/chosen": -0.9185325503349304,
      "logps/rejected": -0.9634258151054382,
      "loss": 1.587,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.9185325503349304,
      "rewards/margins": 0.044893164187669754,
      "rewards/rejected": -0.9634258151054382,
      "step": 55
    },
    {
      "epoch": 0.12860013395847286,
      "grad_norm": 29.275115325498533,
      "learning_rate": 1.9952533792092286e-05,
      "logits/chosen": -2.7369465827941895,
      "logits/rejected": -2.746910810470581,
      "logps/chosen": -0.8204312324523926,
      "logps/rejected": -0.871277928352356,
      "loss": 1.6101,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.8204312324523926,
      "rewards/margins": 0.05084659531712532,
      "rewards/rejected": -0.871277928352356,
      "step": 60
    },
    {
      "epoch": 0.13931681178834562,
      "grad_norm": 9.013295050879966,
      "learning_rate": 1.9909065766585522e-05,
      "logits/chosen": -2.476846694946289,
      "logits/rejected": -2.4859910011291504,
      "logps/chosen": -0.8948932886123657,
      "logps/rejected": -0.9379276037216187,
      "loss": 1.6324,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.8948932886123657,
      "rewards/margins": 0.043034326285123825,
      "rewards/rejected": -0.9379276037216187,
      "step": 65
    },
    {
      "epoch": 0.15003348961821836,
      "grad_norm": 8.308683516584036,
      "learning_rate": 1.9851672793982618e-05,
      "logits/chosen": -2.443450450897217,
      "logits/rejected": -2.465125799179077,
      "logps/chosen": -0.957764744758606,
      "logps/rejected": -0.9942213296890259,
      "loss": 1.6029,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.957764744758606,
      "rewards/margins": 0.03645653277635574,
      "rewards/rejected": -0.9942213296890259,
      "step": 70
    },
    {
      "epoch": 0.1607501674480911,
      "grad_norm": 7.915844945947729,
      "learning_rate": 1.978043552710454e-05,
      "logits/chosen": -2.4849705696105957,
      "logits/rejected": -2.4826462268829346,
      "logps/chosen": -0.9050397872924805,
      "logps/rejected": -0.9400200843811035,
      "loss": 1.6052,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.9050397872924805,
      "rewards/margins": 0.03498036414384842,
      "rewards/rejected": -0.9400200843811035,
      "step": 75
    },
    {
      "epoch": 0.17146684527796383,
      "grad_norm": 304.0629881690128,
      "learning_rate": 1.9695454073791517e-05,
      "logits/chosen": -2.500096559524536,
      "logits/rejected": -2.5168652534484863,
      "logps/chosen": -0.906775951385498,
      "logps/rejected": -1.02707040309906,
      "loss": 1.82,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.906775951385498,
      "rewards/margins": 0.12029434740543365,
      "rewards/rejected": -1.02707040309906,
      "step": 80
    },
    {
      "epoch": 0.18218352310783656,
      "grad_norm": 10.960148765645817,
      "learning_rate": 1.9596847856224143e-05,
      "logits/chosen": -2.257906436920166,
      "logits/rejected": -2.265631675720215,
      "logps/chosen": -1.1177845001220703,
      "logps/rejected": -1.1891874074935913,
      "loss": 1.6996,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.1177845001220703,
      "rewards/margins": 0.07140282541513443,
      "rewards/rejected": -1.1891874074935913,
      "step": 85
    },
    {
      "epoch": 0.19290020093770932,
      "grad_norm": 10.383486663685009,
      "learning_rate": 1.9484755443102573e-05,
      "logits/chosen": -2.215013027191162,
      "logits/rejected": -2.2123208045959473,
      "logps/chosen": -1.035982370376587,
      "logps/rejected": -1.0804064273834229,
      "loss": 1.6576,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.035982370376587,
      "rewards/margins": 0.044424109160900116,
      "rewards/rejected": -1.0804064273834229,
      "step": 90
    },
    {
      "epoch": 0.20361687876758205,
      "grad_norm": 9.444434646488055,
      "learning_rate": 1.9359334354919587e-05,
      "logits/chosen": -2.5854439735412598,
      "logits/rejected": -2.604400157928467,
      "logps/chosen": -1.0645908117294312,
      "logps/rejected": -1.1030633449554443,
      "loss": 1.6255,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.0645908117294312,
      "rewards/margins": 0.03847253695130348,
      "rewards/rejected": -1.1030633449554443,
      "step": 95
    },
    {
      "epoch": 0.2143335565974548,
      "grad_norm": 10.008429713827578,
      "learning_rate": 1.9220760842601245e-05,
      "logits/chosen": -2.41093373298645,
      "logits/rejected": -2.416090488433838,
      "logps/chosen": -1.0662959814071655,
      "logps/rejected": -1.0942766666412354,
      "loss": 1.6366,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.0662959814071655,
      "rewards/margins": 0.027980733662843704,
      "rewards/rejected": -1.0942766666412354,
      "step": 100
    },
    {
      "epoch": 0.22505023442732752,
      "grad_norm": 13.793372325395232,
      "learning_rate": 1.9069229639826113e-05,
      "logits/chosen": -2.593897581100464,
      "logits/rejected": -2.592832326889038,
      "logps/chosen": -1.0412575006484985,
      "logps/rejected": -1.1209934949874878,
      "loss": 1.6663,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.0412575006484985,
      "rewards/margins": 0.0797361508011818,
      "rewards/rejected": -1.1209934949874878,
      "step": 105
    },
    {
      "epoch": 0.23576691225720026,
      "grad_norm": 25.707371820916816,
      "learning_rate": 1.8904953689371223e-05,
      "logits/chosen": -2.1585533618927,
      "logits/rejected": -2.143022060394287,
      "logps/chosen": -1.0903128385543823,
      "logps/rejected": -1.1769222021102905,
      "loss": 1.6708,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0903128385543823,
      "rewards/margins": 0.08660950511693954,
      "rewards/rejected": -1.1769222021102905,
      "step": 110
    },
    {
      "epoch": 0.24648359008707302,
      "grad_norm": 15.08396540818952,
      "learning_rate": 1.8728163843869215e-05,
      "logits/chosen": -2.3684897422790527,
      "logits/rejected": -2.3717474937438965,
      "logps/chosen": -1.1836369037628174,
      "logps/rejected": -1.1988976001739502,
      "loss": 1.8024,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.1836369037628174,
      "rewards/margins": 0.015260684303939342,
      "rewards/rejected": -1.1988976001739502,
      "step": 115
    },
    {
      "epoch": 0.2572002679169457,
      "grad_norm": 10.565734009587494,
      "learning_rate": 1.853910854139727e-05,
      "logits/chosen": -2.122260570526123,
      "logits/rejected": -2.138091802597046,
      "logps/chosen": -1.133177399635315,
      "logps/rejected": -1.1549627780914307,
      "loss": 1.7056,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.133177399635315,
      "rewards/margins": 0.02178535796701908,
      "rewards/rejected": -1.1549627780914307,
      "step": 120
    },
    {
      "epoch": 0.2679169457468185,
      "grad_norm": 7.929268982931603,
      "learning_rate": 1.833805345635369e-05,
      "logits/chosen": -2.467066526412964,
      "logits/rejected": -2.470454692840576,
      "logps/chosen": -1.0294818878173828,
      "logps/rejected": -1.0639444589614868,
      "loss": 1.6829,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.0294818878173828,
      "rewards/margins": 0.03446248173713684,
      "rewards/rejected": -1.0639444589614868,
      "step": 125
    },
    {
      "epoch": 0.27863362357669125,
      "grad_norm": 9.40947063841382,
      "learning_rate": 1.8125281126112678e-05,
      "logits/chosen": -2.5180554389953613,
      "logits/rejected": -2.5261032581329346,
      "logps/chosen": -1.044611930847168,
      "logps/rejected": -1.1444398164749146,
      "loss": 1.6283,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.044611930847168,
      "rewards/margins": 0.09982797503471375,
      "rewards/rejected": -1.1444398164749146,
      "step": 130
    },
    {
      "epoch": 0.289350301406564,
      "grad_norm": 7.797174649236775,
      "learning_rate": 1.790109055398212e-05,
      "logits/chosen": -2.5548009872436523,
      "logits/rejected": -2.5629782676696777,
      "logps/chosen": -0.9959312677383423,
      "logps/rejected": -1.0395903587341309,
      "loss": 1.6327,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9959312677383423,
      "rewards/margins": 0.04365905374288559,
      "rewards/rejected": -1.0395903587341309,
      "step": 135
    },
    {
      "epoch": 0.3000669792364367,
      "grad_norm": 8.65910654888172,
      "learning_rate": 1.766579678902212e-05,
      "logits/chosen": -2.452951431274414,
      "logits/rejected": -2.459916114807129,
      "logps/chosen": -0.9893513917922974,
      "logps/rejected": -1.0323150157928467,
      "loss": 1.6058,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9893513917922974,
      "rewards/margins": 0.042963750660419464,
      "rewards/rejected": -1.0323150157928467,
      "step": 140
    },
    {
      "epoch": 0.31078365706630945,
      "grad_norm": 8.475342441948424,
      "learning_rate": 1.741973048331496e-05,
      "logits/chosen": -2.608076810836792,
      "logits/rejected": -2.592602491378784,
      "logps/chosen": -0.959572434425354,
      "logps/rejected": -1.034380316734314,
      "loss": 1.5962,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.959572434425354,
      "rewards/margins": 0.07480800151824951,
      "rewards/rejected": -1.034380316734314,
      "step": 145
    },
    {
      "epoch": 0.3215003348961822,
      "grad_norm": 13.2413080829705,
      "learning_rate": 1.7163237427308477e-05,
      "logits/chosen": -2.6177635192871094,
      "logits/rejected": -2.6168532371520996,
      "logps/chosen": -1.004525899887085,
      "logps/rejected": -1.0910793542861938,
      "loss": 1.6239,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.004525899887085,
      "rewards/margins": 0.08655332773923874,
      "rewards/rejected": -1.0910793542861938,
      "step": 150
    },
    {
      "epoch": 0.3322170127260549,
      "grad_norm": 8.151298227822702,
      "learning_rate": 1.689667806388594e-05,
      "logits/chosen": -2.7449164390563965,
      "logits/rejected": -2.7318592071533203,
      "logps/chosen": -0.9720630645751953,
      "logps/rejected": -1.047321081161499,
      "loss": 1.5985,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.9720630645751953,
      "rewards/margins": 0.0752580463886261,
      "rewards/rejected": -1.047321081161499,
      "step": 155
    },
    {
      "epoch": 0.34293369055592765,
      "grad_norm": 7.660153117545113,
      "learning_rate": 1.6620426981845233e-05,
      "logits/chosen": -2.5920422077178955,
      "logits/rejected": -2.5721192359924316,
      "logps/chosen": -0.9702916145324707,
      "logps/rejected": -1.0599411725997925,
      "loss": 1.6107,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.9702916145324707,
      "rewards/margins": 0.08964961767196655,
      "rewards/rejected": -1.0599411725997925,
      "step": 160
    },
    {
      "epoch": 0.3536503683858004,
      "grad_norm": 469.42247386416926,
      "learning_rate": 1.633487238949915e-05,
      "logits/chosen": -2.528369426727295,
      "logits/rejected": -2.5209622383117676,
      "logps/chosen": -0.9299852252006531,
      "logps/rejected": -0.9966603517532349,
      "loss": 1.6799,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.9299852252006531,
      "rewards/margins": 0.06667519360780716,
      "rewards/rejected": -0.9966603517532349,
      "step": 165
    },
    {
      "epoch": 0.3643670462156731,
      "grad_norm": 13.123351141683822,
      "learning_rate": 1.6040415569136586e-05,
      "logits/chosen": -2.3123598098754883,
      "logits/rejected": -2.3209071159362793,
      "logps/chosen": -1.2778890132904053,
      "logps/rejected": -1.3249436616897583,
      "loss": 1.7418,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.2778890132904053,
      "rewards/margins": 0.04705450311303139,
      "rewards/rejected": -1.3249436616897583,
      "step": 170
    },
    {
      "epoch": 0.3750837240455459,
      "grad_norm": 7.9321661768835865,
      "learning_rate": 1.5737470313111134e-05,
      "logits/chosen": -2.1733322143554688,
      "logits/rejected": -2.188865900039673,
      "logps/chosen": -1.1108664274215698,
      "logps/rejected": -1.1735206842422485,
      "loss": 1.6869,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.1108664274215698,
      "rewards/margins": 0.06265412271022797,
      "rewards/rejected": -1.1735206842422485,
      "step": 175
    },
    {
      "epoch": 0.38580040187541864,
      "grad_norm": 8.557996815044758,
      "learning_rate": 1.5426462342349736e-05,
      "logits/chosen": -2.3044323921203613,
      "logits/rejected": -2.3116490840911865,
      "logps/chosen": -1.1034280061721802,
      "logps/rejected": -1.1603246927261353,
      "loss": 1.6567,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.1034280061721802,
      "rewards/margins": 0.05689678341150284,
      "rewards/rejected": -1.1603246927261353,
      "step": 180
    },
    {
      "epoch": 0.3965170797052914,
      "grad_norm": 10.818770194479542,
      "learning_rate": 1.5107828708098327e-05,
      "logits/chosen": -2.3942196369171143,
      "logits/rejected": -2.4129745960235596,
      "logps/chosen": -1.0460340976715088,
      "logps/rejected": -1.105322241783142,
      "loss": 1.6177,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0460340976715088,
      "rewards/margins": 0.059288132935762405,
      "rewards/rejected": -1.105322241783142,
      "step": 185
    },
    {
      "epoch": 0.4072337575351641,
      "grad_norm": 15.512656922129596,
      "learning_rate": 1.4782017177745322e-05,
      "logits/chosen": -2.4357337951660156,
      "logits/rejected": -2.458134889602661,
      "logps/chosen": -1.0308730602264404,
      "logps/rejected": -1.2367624044418335,
      "loss": 1.6917,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.0308730602264404,
      "rewards/margins": 0.20588929951190948,
      "rewards/rejected": -1.2367624044418335,
      "step": 190
    },
    {
      "epoch": 0.41795043536503684,
      "grad_norm": 10.838461128892558,
      "learning_rate": 1.444948560558601e-05,
      "logits/chosen": -2.4340341091156006,
      "logits/rejected": -2.4456231594085693,
      "logps/chosen": -1.2891473770141602,
      "logps/rejected": -1.3041666746139526,
      "loss": 1.7583,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.2891473770141602,
      "rewards/margins": 0.015019327402114868,
      "rewards/rejected": -1.3041666746139526,
      "step": 195
    },
    {
      "epoch": 0.4286671131949096,
      "grad_norm": 8.28412245267914,
      "learning_rate": 1.4110701289412045e-05,
      "logits/chosen": -2.515929937362671,
      "logits/rejected": -2.5307044982910156,
      "logps/chosen": -1.1040905714035034,
      "logps/rejected": -1.1781259775161743,
      "loss": 1.6734,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.1040905714035034,
      "rewards/margins": 0.07403533160686493,
      "rewards/rejected": -1.1781259775161743,
      "step": 200
    },
    {
      "epoch": 0.4393837910247823,
      "grad_norm": 9.96354155003577,
      "learning_rate": 1.3766140313830279e-05,
      "logits/chosen": -2.531589984893799,
      "logits/rejected": -2.520904302597046,
      "logps/chosen": -1.0145483016967773,
      "logps/rejected": -1.0664831399917603,
      "loss": 1.6419,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.0145483016967773,
      "rewards/margins": 0.05193489044904709,
      "rewards/rejected": -1.0664831399917603,
      "step": 205
    },
    {
      "epoch": 0.45010046885465504,
      "grad_norm": 7.434219003968733,
      "learning_rate": 1.341628688123369e-05,
      "logits/chosen": -2.517782211303711,
      "logits/rejected": -2.5262608528137207,
      "logps/chosen": -1.0025173425674438,
      "logps/rejected": -1.0794353485107422,
      "loss": 1.606,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.0025173425674438,
      "rewards/margins": 0.07691796869039536,
      "rewards/rejected": -1.0794353485107422,
      "step": 210
    },
    {
      "epoch": 0.4608171466845278,
      "grad_norm": 7.337174471827815,
      "learning_rate": 1.3061632631364613e-05,
      "logits/chosen": -2.5459511280059814,
      "logits/rejected": -2.546247959136963,
      "logps/chosen": -0.9931095242500305,
      "logps/rejected": -1.094226598739624,
      "loss": 1.594,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.9931095242500305,
      "rewards/margins": 0.10111699998378754,
      "rewards/rejected": -1.094226598739624,
      "step": 215
    },
    {
      "epoch": 0.4715338245144005,
      "grad_norm": 8.701123471362035,
      "learning_rate": 1.2702675950426477e-05,
      "logits/chosen": -2.7032644748687744,
      "logits/rejected": -2.685612440109253,
      "logps/chosen": -0.9561999440193176,
      "logps/rejected": -1.0159614086151123,
      "loss": 1.63,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.9561999440193176,
      "rewards/margins": 0.05976147577166557,
      "rewards/rejected": -1.0159614086151123,
      "step": 220
    },
    {
      "epoch": 0.4822505023442733,
      "grad_norm": 10.855107466518529,
      "learning_rate": 1.2339921270714873e-05,
      "logits/chosen": -2.7168991565704346,
      "logits/rejected": -2.742966651916504,
      "logps/chosen": -0.9448912739753723,
      "logps/rejected": -1.0768978595733643,
      "loss": 1.5893,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.9448912739753723,
      "rewards/margins": 0.1320066601037979,
      "rewards/rejected": -1.0768978595733643,
      "step": 225
    },
    {
      "epoch": 0.49296718017414604,
      "grad_norm": 7.921384206654596,
      "learning_rate": 1.1973878361752263e-05,
      "logits/chosen": -2.7181243896484375,
      "logits/rejected": -2.7184972763061523,
      "logps/chosen": -0.9301810264587402,
      "logps/rejected": -1.000950813293457,
      "loss": 1.5781,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.9301810264587402,
      "rewards/margins": 0.07076982408761978,
      "rewards/rejected": -1.000950813293457,
      "step": 230
    },
    {
      "epoch": 0.5036838580040187,
      "grad_norm": 8.000512303019988,
      "learning_rate": 1.160506161392239e-05,
      "logits/chosen": -2.7740559577941895,
      "logits/rejected": -2.767812967300415,
      "logps/chosen": -0.906406044960022,
      "logps/rejected": -1.006753921508789,
      "loss": 1.5802,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.906406044960022,
      "rewards/margins": 0.10034787654876709,
      "rewards/rejected": -1.006753921508789,
      "step": 235
    },
    {
      "epoch": 0.5144005358338914,
      "grad_norm": 9.398458173467265,
      "learning_rate": 1.1233989315611132e-05,
      "logits/chosen": -2.792450428009033,
      "logits/rejected": -2.7874622344970703,
      "logps/chosen": -0.9394813776016235,
      "logps/rejected": -1.022139549255371,
      "loss": 1.587,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.9394813776016235,
      "rewards/margins": 0.08265805244445801,
      "rewards/rejected": -1.022139549255371,
      "step": 240
    },
    {
      "epoch": 0.5251172136637642,
      "grad_norm": 10.738263842024553,
      "learning_rate": 1.0861182924869584e-05,
      "logits/chosen": -2.721609115600586,
      "logits/rejected": -2.712308406829834,
      "logps/chosen": -0.9511491656303406,
      "logps/rejected": -1.0187407732009888,
      "loss": 1.5773,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.9511491656303406,
      "rewards/margins": 0.06759153306484222,
      "rewards/rejected": -1.0187407732009888,
      "step": 245
    },
    {
      "epoch": 0.535833891493637,
      "grad_norm": 8.609658733354994,
      "learning_rate": 1.0487166336622903e-05,
      "logits/chosen": -2.714517116546631,
      "logits/rejected": -2.7152135372161865,
      "logps/chosen": -0.9705566167831421,
      "logps/rejected": -1.0084294080734253,
      "loss": 1.5971,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.9705566167831421,
      "rewards/margins": 0.03787278011441231,
      "rewards/rejected": -1.0084294080734253,
      "step": 250
    },
    {
      "epoch": 0.5465505693235098,
      "grad_norm": 8.407135051661365,
      "learning_rate": 1.0112465146454681e-05,
      "logits/chosen": -2.790414333343506,
      "logits/rejected": -2.799837112426758,
      "logps/chosen": -0.8985916376113892,
      "logps/rejected": -0.9925104379653931,
      "loss": 1.5696,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.8985916376113892,
      "rewards/margins": 0.09391861408948898,
      "rewards/rejected": -0.9925104379653931,
      "step": 255
    },
    {
      "epoch": 0.5572672471533825,
      "grad_norm": 8.166210902327252,
      "learning_rate": 9.737605912001413e-06,
      "logits/chosen": -2.7387256622314453,
      "logits/rejected": -2.7400269508361816,
      "logps/chosen": -0.9064778089523315,
      "logps/rejected": -1.0061310529708862,
      "loss": 1.5824,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.9064778089523315,
      "rewards/margins": 0.09965313225984573,
      "rewards/rejected": -1.0061310529708862,
      "step": 260
    },
    {
      "epoch": 0.5679839249832552,
      "grad_norm": 7.963553030396463,
      "learning_rate": 9.363115412995038e-06,
      "logits/chosen": -2.733085870742798,
      "logits/rejected": -2.7209997177124023,
      "logps/chosen": -0.9366081357002258,
      "logps/rejected": -1.074143648147583,
      "loss": 1.5601,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9366081357002258,
      "rewards/margins": 0.13753557205200195,
      "rewards/rejected": -1.074143648147583,
      "step": 265
    },
    {
      "epoch": 0.578700602813128,
      "grad_norm": 8.394463914962087,
      "learning_rate": 8.989519910993357e-06,
      "logits/chosen": -2.741532802581787,
      "logits/rejected": -2.7389678955078125,
      "logps/chosen": -0.8945878744125366,
      "logps/rejected": -0.9697369337081909,
      "loss": 1.5652,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.8945878744125366,
      "rewards/margins": 0.07514907419681549,
      "rewards/rejected": -0.9697369337081909,
      "step": 270
    },
    {
      "epoch": 0.5894172806430007,
      "grad_norm": 7.2839753266908875,
      "learning_rate": 8.617344409838631e-06,
      "logits/chosen": -2.837791919708252,
      "logits/rejected": -2.8426520824432373,
      "logps/chosen": -0.8537940979003906,
      "logps/rejected": -0.9409109354019165,
      "loss": 1.5685,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.8537940979003906,
      "rewards/margins": 0.08711685240268707,
      "rewards/rejected": -0.9409109354019165,
      "step": 275
    },
    {
      "epoch": 0.6001339584728734,
      "grad_norm": 7.994187957031587,
      "learning_rate": 8.247111917883634e-06,
      "logits/chosen": -2.6431148052215576,
      "logits/rejected": -2.644808530807495,
      "logps/chosen": -0.8992069959640503,
      "logps/rejected": -1.0336244106292725,
      "loss": 1.5658,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.8992069959640503,
      "rewards/margins": 0.1344173550605774,
      "rewards/rejected": -1.0336244106292725,
      "step": 280
    },
    {
      "epoch": 0.6108506363027462,
      "grad_norm": 10.3254576486645,
      "learning_rate": 7.879342713021875e-06,
      "logits/chosen": -2.745375156402588,
      "logits/rejected": -2.744263172149658,
      "logps/chosen": -0.9114044308662415,
      "logps/rejected": -0.9392461776733398,
      "loss": 1.5753,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.9114044308662415,
      "rewards/margins": 0.027841681614518166,
      "rewards/rejected": -0.9392461776733398,
      "step": 285
    },
    {
      "epoch": 0.6215673141326189,
      "grad_norm": 14.256488881900221,
      "learning_rate": 7.514553611554909e-06,
      "logits/chosen": -2.9054572582244873,
      "logits/rejected": -2.915325880050659,
      "logps/chosen": -0.8290088772773743,
      "logps/rejected": -0.9459854364395142,
      "loss": 1.5614,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.8290088772773743,
      "rewards/margins": 0.11697645485401154,
      "rewards/rejected": -0.9459854364395142,
      "step": 290
    },
    {
      "epoch": 0.6322839919624916,
      "grad_norm": 8.049072856223585,
      "learning_rate": 7.15325724192409e-06,
      "logits/chosen": -2.861290454864502,
      "logits/rejected": -2.8629579544067383,
      "logps/chosen": -0.8957312703132629,
      "logps/rejected": -1.0799039602279663,
      "loss": 1.5436,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.8957312703132629,
      "rewards/margins": 0.1841726452112198,
      "rewards/rejected": -1.0799039602279663,
      "step": 295
    },
    {
      "epoch": 0.6430006697923644,
      "grad_norm": 8.353215812366958,
      "learning_rate": 6.79596132432742e-06,
      "logits/chosen": -2.8521814346313477,
      "logits/rejected": -2.8686671257019043,
      "logps/chosen": -0.8559802770614624,
      "logps/rejected": -0.9652966260910034,
      "loss": 1.5273,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.8559802770614624,
      "rewards/margins": 0.10931621491909027,
      "rewards/rejected": -0.9652966260910034,
      "step": 300
    },
    {
      "epoch": 0.6537173476222371,
      "grad_norm": 8.558746762469774,
      "learning_rate": 6.443167957233864e-06,
      "logits/chosen": -2.8060519695281982,
      "logits/rejected": -2.793569326400757,
      "logps/chosen": -0.8676208257675171,
      "logps/rejected": -0.9558470845222473,
      "loss": 1.5333,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8676208257675171,
      "rewards/margins": 0.08822615444660187,
      "rewards/rejected": -0.9558470845222473,
      "step": 305
    },
    {
      "epoch": 0.6644340254521098,
      "grad_norm": 8.944155497280653,
      "learning_rate": 6.095372911797673e-06,
      "logits/chosen": -2.8632490634918213,
      "logits/rejected": -2.859429359436035,
      "logps/chosen": -0.8489475250244141,
      "logps/rejected": -0.9311763644218445,
      "loss": 1.5397,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8489475250244141,
      "rewards/margins": 0.08222876489162445,
      "rewards/rejected": -0.9311763644218445,
      "step": 310
    },
    {
      "epoch": 0.6751507032819826,
      "grad_norm": 7.822389173887281,
      "learning_rate": 5.753064935164348e-06,
      "logits/chosen": -2.7866902351379395,
      "logits/rejected": -2.792357921600342,
      "logps/chosen": -0.84296715259552,
      "logps/rejected": -0.959184467792511,
      "loss": 1.5381,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.84296715259552,
      "rewards/margins": 0.11621735244989395,
      "rewards/rejected": -0.959184467792511,
      "step": 315
    },
    {
      "epoch": 0.6858673811118553,
      "grad_norm": 9.32294397022148,
      "learning_rate": 5.4167250636472426e-06,
      "logits/chosen": -2.7923617362976074,
      "logits/rejected": -2.7717807292938232,
      "logps/chosen": -0.8463284373283386,
      "logps/rejected": -0.9291221499443054,
      "loss": 1.5458,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8463284373283386,
      "rewards/margins": 0.08279372751712799,
      "rewards/rejected": -0.9291221499443054,
      "step": 320
    },
    {
      "epoch": 0.696584058941728,
      "grad_norm": 8.808118982464391,
      "learning_rate": 5.08682594674002e-06,
      "logits/chosen": -2.7373509407043457,
      "logits/rejected": -2.7527077198028564,
      "logps/chosen": -0.8056324124336243,
      "logps/rejected": -0.957785964012146,
      "loss": 1.5196,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8056324124336243,
      "rewards/margins": 0.15215365588665009,
      "rewards/rejected": -0.957785964012146,
      "step": 325
    },
    {
      "epoch": 0.7073007367716008,
      "grad_norm": 8.169267761293227,
      "learning_rate": 4.763831182914848e-06,
      "logits/chosen": -2.7620842456817627,
      "logits/rejected": -2.77044415473938,
      "logps/chosen": -0.8720673322677612,
      "logps/rejected": -0.9474166631698608,
      "loss": 1.5373,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.8720673322677612,
      "rewards/margins": 0.07534926384687424,
      "rewards/rejected": -0.9474166631698608,
      "step": 330
    },
    {
      "epoch": 0.7180174146014735,
      "grad_norm": 8.7345666789259,
      "learning_rate": 4.448194668139777e-06,
      "logits/chosen": -2.794130325317383,
      "logits/rejected": -2.7945706844329834,
      "logps/chosen": -0.8377755284309387,
      "logps/rejected": -0.9672821760177612,
      "loss": 1.5233,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8377755284309387,
      "rewards/margins": 0.12950672209262848,
      "rewards/rejected": -0.9672821760177612,
      "step": 335
    },
    {
      "epoch": 0.7287340924313462,
      "grad_norm": 8.389766148627231,
      "learning_rate": 4.1403599580307795e-06,
      "logits/chosen": -2.714247226715088,
      "logits/rejected": -2.7236764430999756,
      "logps/chosen": -0.8354495167732239,
      "logps/rejected": -0.9634197354316711,
      "loss": 1.5335,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.8354495167732239,
      "rewards/margins": 0.1279701441526413,
      "rewards/rejected": -0.9634197354316711,
      "step": 340
    },
    {
      "epoch": 0.739450770261219,
      "grad_norm": 9.147040621292957,
      "learning_rate": 3.840759644534828e-06,
      "logits/chosen": -2.790487766265869,
      "logits/rejected": -2.8023035526275635,
      "logps/chosen": -0.8659473657608032,
      "logps/rejected": -0.9856260418891907,
      "loss": 1.5348,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.8659473657608032,
      "rewards/margins": 0.11967878043651581,
      "rewards/rejected": -0.9856260418891907,
      "step": 345
    },
    {
      "epoch": 0.7501674480910918,
      "grad_norm": 8.599069402922064,
      "learning_rate": 3.549814748019906e-06,
      "logits/chosen": -2.815507411956787,
      "logits/rejected": -2.824169158935547,
      "logps/chosen": -0.808633029460907,
      "logps/rejected": -0.9542997479438782,
      "loss": 1.529,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.808633029460907,
      "rewards/margins": 0.14566662907600403,
      "rewards/rejected": -0.9542997479438782,
      "step": 350
    },
    {
      "epoch": 0.7608841259209645,
      "grad_norm": 7.982319756803084,
      "learning_rate": 3.2679341256262653e-06,
      "logits/chosen": -2.8472940921783447,
      "logits/rejected": -2.8238396644592285,
      "logps/chosen": -0.9044294357299805,
      "logps/rejected": -0.9391825795173645,
      "loss": 1.5418,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.9044294357299805,
      "rewards/margins": 0.03475295007228851,
      "rewards/rejected": -0.9391825795173645,
      "step": 355
    },
    {
      "epoch": 0.7716008037508373,
      "grad_norm": 11.32319755235924,
      "learning_rate": 2.995513896710327e-06,
      "logits/chosen": -2.8410215377807617,
      "logits/rejected": -2.83413028717041,
      "logps/chosen": -0.7960796356201172,
      "logps/rejected": -0.9083698391914368,
      "loss": 1.5306,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.7960796356201172,
      "rewards/margins": 0.1122901439666748,
      "rewards/rejected": -0.9083698391914368,
      "step": 360
    },
    {
      "epoch": 0.78231748158071,
      "grad_norm": 7.8205965789155325,
      "learning_rate": 2.7329368861886862e-06,
      "logits/chosen": -2.8469526767730713,
      "logits/rejected": -2.8535444736480713,
      "logps/chosen": -0.8426831960678101,
      "logps/rejected": -0.9201619029045105,
      "loss": 1.5523,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8426831960678101,
      "rewards/margins": 0.07747860997915268,
      "rewards/rejected": -0.9201619029045105,
      "step": 365
    },
    {
      "epoch": 0.7930341594105828,
      "grad_norm": 11.791063151570222,
      "learning_rate": 2.4805720865643943e-06,
      "logits/chosen": -2.8243472576141357,
      "logits/rejected": -2.8304176330566406,
      "logps/chosen": -0.8739852905273438,
      "logps/rejected": -0.9622358083724976,
      "loss": 1.5445,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.8739852905273438,
      "rewards/margins": 0.08825065940618515,
      "rewards/rejected": -0.9622358083724976,
      "step": 370
    },
    {
      "epoch": 0.8037508372404555,
      "grad_norm": 10.592008554970972,
      "learning_rate": 2.2387741393916117e-06,
      "logits/chosen": -2.7737154960632324,
      "logits/rejected": -2.7636284828186035,
      "logps/chosen": -0.9277487993240356,
      "logps/rejected": -1.0097343921661377,
      "loss": 1.5368,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.9277487993240356,
      "rewards/margins": 0.08198549598455429,
      "rewards/rejected": -1.0097343921661377,
      "step": 375
    },
    {
      "epoch": 0.8144675150703282,
      "grad_norm": 8.911271573494423,
      "learning_rate": 2.0078828369072323e-06,
      "logits/chosen": -2.7896554470062256,
      "logits/rejected": -2.770059823989868,
      "logps/chosen": -0.8297151327133179,
      "logps/rejected": -0.9309379458427429,
      "loss": 1.531,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.8297151327133179,
      "rewards/margins": 0.10122283548116684,
      "rewards/rejected": -0.9309379458427429,
      "step": 380
    },
    {
      "epoch": 0.825184192900201,
      "grad_norm": 10.364577364972547,
      "learning_rate": 1.7882226445298656e-06,
      "logits/chosen": -2.740999460220337,
      "logits/rejected": -2.733097791671753,
      "logps/chosen": -0.8539925813674927,
      "logps/rejected": -0.9543533325195312,
      "loss": 1.5174,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8539925813674927,
      "rewards/margins": 0.10036077350378036,
      "rewards/rejected": -0.9543533325195312,
      "step": 385
    },
    {
      "epoch": 0.8359008707300737,
      "grad_norm": 10.862441952775951,
      "learning_rate": 1.580102244897217e-06,
      "logits/chosen": -2.7668397426605225,
      "logits/rejected": -2.7470200061798096,
      "logps/chosen": -0.8713840246200562,
      "logps/rejected": -0.940260112285614,
      "loss": 1.5632,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.8713840246200562,
      "rewards/margins": 0.0688760057091713,
      "rewards/rejected": -0.940260112285614,
      "step": 390
    },
    {
      "epoch": 0.8466175485599464,
      "grad_norm": 7.998296949983092,
      "learning_rate": 1.3838141040825536e-06,
      "logits/chosen": -2.85998272895813,
      "logits/rejected": -2.8610522747039795,
      "logps/chosen": -0.8187233805656433,
      "logps/rejected": -0.8954393267631531,
      "loss": 1.5253,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.8187233805656433,
      "rewards/margins": 0.0767158791422844,
      "rewards/rejected": -0.8954393267631531,
      "step": 395
    },
    {
      "epoch": 0.8573342263898192,
      "grad_norm": 10.009174586154066,
      "learning_rate": 1.1996340605999058e-06,
      "logits/chosen": -2.842839479446411,
      "logits/rejected": -2.819075107574463,
      "logps/chosen": -0.7899017333984375,
      "logps/rejected": -0.9768079519271851,
      "loss": 1.5218,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.7899017333984375,
      "rewards/margins": 0.18690629303455353,
      "rewards/rejected": -0.9768079519271851,
      "step": 400
    },
    {
      "epoch": 0.8573342263898192,
      "eval_logits/chosen": -2.840418815612793,
      "eval_logits/rejected": -2.8387703895568848,
      "eval_logps/chosen": -0.8695030808448792,
      "eval_logps/rejected": -0.9116575121879578,
      "eval_loss": 1.5846024751663208,
      "eval_rewards/accuracies": 0.5558510422706604,
      "eval_rewards/chosen": -0.8695030808448792,
      "eval_rewards/margins": 0.0421544648706913,
      "eval_rewards/rejected": -0.9116575121879578,
      "eval_runtime": 432.5597,
      "eval_samples_per_second": 6.922,
      "eval_steps_per_second": 0.435,
      "step": 400
    },
    {
      "epoch": 0.8680509042196919,
      "grad_norm": 8.778213042419416,
      "learning_rate": 1.0278209377755099e-06,
      "logits/chosen": -2.829361915588379,
      "logits/rejected": -2.8452582359313965,
      "logps/chosen": -0.8199126124382019,
      "logps/rejected": -0.9467391967773438,
      "loss": 1.5156,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.8199126124382019,
      "rewards/margins": 0.12682652473449707,
      "rewards/rejected": -0.9467391967773438,
      "step": 405
    },
    {
      "epoch": 0.8787675820495646,
      "grad_norm": 8.779487119016105,
      "learning_rate": 8.68616180030275e-07,
      "logits/chosen": -2.865104913711548,
      "logits/rejected": -2.867384672164917,
      "logps/chosen": -0.8341816067695618,
      "logps/rejected": -0.8998939394950867,
      "loss": 1.5114,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.8341816067695618,
      "rewards/margins": 0.06571228057146072,
      "rewards/rejected": -0.8998939394950867,
      "step": 410
    },
    {
      "epoch": 0.8894842598794374,
      "grad_norm": 8.92068989780619,
      "learning_rate": 7.22243513584342e-07,
      "logits/chosen": -2.77596378326416,
      "logits/rejected": -2.7669265270233154,
      "logps/chosen": -0.8306072354316711,
      "logps/rejected": -0.9651764035224915,
      "loss": 1.5034,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8306072354316711,
      "rewards/margins": 0.13456913828849792,
      "rewards/rejected": -0.9651764035224915,
      "step": 415
    },
    {
      "epoch": 0.9002009377093101,
      "grad_norm": 9.219885917035732,
      "learning_rate": 5.889086320605575e-07,
      "logits/chosen": -2.7921142578125,
      "logits/rejected": -2.7921335697174072,
      "logps/chosen": -0.7846695184707642,
      "logps/rejected": -0.8718813061714172,
      "loss": 1.5351,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.7846695184707642,
      "rewards/margins": 0.08721177279949188,
      "rewards/rejected": -0.8718813061714172,
      "step": 420
    },
    {
      "epoch": 0.9109176155391828,
      "grad_norm": 9.448964270757202,
      "learning_rate": 4.6879890742869584e-07,
      "logits/chosen": -2.7913196086883545,
      "logits/rejected": -2.789302110671997,
      "logps/chosen": -0.7865716218948364,
      "logps/rejected": -0.9528288841247559,
      "loss": 1.5108,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.7865716218948364,
      "rewards/margins": 0.16625720262527466,
      "rewards/rejected": -0.9528288841247559,
      "step": 425
    },
    {
      "epoch": 0.9216342933690556,
      "grad_norm": 10.369844004542797,
      "learning_rate": 3.6208312669659405e-07,
      "logits/chosen": -2.817171812057495,
      "logits/rejected": -2.8241400718688965,
      "logps/chosen": -0.8919852375984192,
      "logps/rejected": -1.037209153175354,
      "loss": 1.5259,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.8919852375984192,
      "rewards/margins": 0.14522376656532288,
      "rewards/rejected": -1.037209153175354,
      "step": 430
    },
    {
      "epoch": 0.9323509711989283,
      "grad_norm": 12.055758939525129,
      "learning_rate": 2.689112547182593e-07,
      "logits/chosen": -2.890130043029785,
      "logits/rejected": -2.882509708404541,
      "logps/chosen": -0.7903082370758057,
      "logps/rejected": -0.9467129707336426,
      "loss": 1.5203,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.7903082370758057,
      "rewards/margins": 0.15640470385551453,
      "rewards/rejected": -0.9467129707336426,
      "step": 435
    },
    {
      "epoch": 0.943067649028801,
      "grad_norm": 9.50477372459136,
      "learning_rate": 1.8941422345223448e-07,
      "logits/chosen": -2.7733447551727295,
      "logits/rejected": -2.77803635597229,
      "logps/chosen": -0.7930269241333008,
      "logps/rejected": -0.9639067649841309,
      "loss": 1.4896,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.7930269241333008,
      "rewards/margins": 0.17087996006011963,
      "rewards/rejected": -0.9639067649841309,
      "step": 440
    },
    {
      "epoch": 0.9537843268586738,
      "grad_norm": 10.00023586076567,
      "learning_rate": 1.2370374796639383e-07,
      "logits/chosen": -2.794691324234009,
      "logits/rejected": -2.799956798553467,
      "logps/chosen": -0.8402734994888306,
      "logps/rejected": -0.9617452621459961,
      "loss": 1.5232,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.8402734994888306,
      "rewards/margins": 0.12147176265716553,
      "rewards/rejected": -0.9617452621459961,
      "step": 445
    },
    {
      "epoch": 0.9645010046885466,
      "grad_norm": 9.634395288787493,
      "learning_rate": 7.18721694477209e-08,
      "logits/chosen": -2.8351221084594727,
      "logits/rejected": -2.833106756210327,
      "logps/chosen": -0.8035479784011841,
      "logps/rejected": -0.9219916462898254,
      "loss": 1.5027,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8035479784011841,
      "rewards/margins": 0.11844360828399658,
      "rewards/rejected": -0.9219916462898254,
      "step": 450
    },
    {
      "epoch": 0.9752176825184193,
      "grad_norm": 9.629343705982949,
      "learning_rate": 3.399232543769504e-08,
      "logits/chosen": -2.807852268218994,
      "logits/rejected": -2.815563678741455,
      "logps/chosen": -0.8062378168106079,
      "logps/rejected": -0.9637410044670105,
      "loss": 1.5073,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.8062378168106079,
      "rewards/margins": 0.15750321745872498,
      "rewards/rejected": -0.9637410044670105,
      "step": 455
    },
    {
      "epoch": 0.9859343603482921,
      "grad_norm": 10.887896034397038,
      "learning_rate": 1.011744747562271e-08,
      "logits/chosen": -2.826111316680908,
      "logits/rejected": -2.830981731414795,
      "logps/chosen": -0.8098823428153992,
      "logps/rejected": -0.9950210452079773,
      "loss": 1.5206,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.8098823428153992,
      "rewards/margins": 0.1851385533809662,
      "rewards/rejected": -0.9950210452079773,
      "step": 460
    },
    {
      "epoch": 0.9966510381781648,
      "grad_norm": 10.914112703731178,
      "learning_rate": 2.810862937696612e-10,
      "logits/chosen": -2.813309907913208,
      "logits/rejected": -2.8205251693725586,
      "logps/chosen": -0.8651328086853027,
      "logps/rejected": -0.9807795286178589,
      "loss": 1.5128,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8651328086853027,
      "rewards/margins": 0.11564669758081436,
      "rewards/rejected": -0.9807795286178589,
      "step": 465
    },
    {
      "epoch": 0.9987943737441393,
      "step": 466,
      "total_flos": 0.0,
      "train_loss": 1.5861326811651304,
      "train_runtime": 19371.3426,
      "train_samples_per_second": 3.083,
      "train_steps_per_second": 0.024
    }
  ],
  "logging_steps": 5,
  "max_steps": 466,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}