{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 32,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "debug/policy_chosen_logits": 26.208711624145508,
      "debug/policy_chosen_logps": -419.32049560546875,
      "debug/policy_rejected_logits": 27.114166259765625,
      "debug/policy_rejected_logps": -409.2409362792969,
      "debug/reference_chosen_logps": -419.32049560546875,
      "debug/reference_rejected_logps": -409.2409362792969,
      "epoch": 0.03125,
      "grad_norm": 5.94402702532091,
      "learning_rate": 1e-06,
      "logits/chosen": 26.208711624145508,
      "logits/rejected": 27.114166259765625,
      "logps/chosen": -419.32049560546875,
      "logps/rejected": -409.2409362792969,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "debug/policy_chosen_logits": 27.364856719970703,
      "debug/policy_chosen_logps": -384.8145446777344,
      "debug/policy_rejected_logits": 30.614818572998047,
      "debug/policy_rejected_logps": -389.1242980957031,
      "debug/reference_chosen_logps": -385.05755615234375,
      "debug/reference_rejected_logps": -389.2853698730469,
      "epoch": 0.0625,
      "grad_norm": 5.403682848041972,
      "learning_rate": 1e-06,
      "logits/chosen": 27.364856719970703,
      "logits/rejected": 30.614818572998047,
      "logps/chosen": -384.8145446777344,
      "logps/rejected": -389.1242980957031,
      "loss": 0.5005,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002430190797895193,
      "rewards/margins": 0.0008192063542082906,
      "rewards/rejected": 0.001610984792932868,
      "step": 2
    },
    {
      "debug/policy_chosen_logits": 27.064964294433594,
      "debug/policy_chosen_logps": -393.77032470703125,
      "debug/policy_rejected_logits": 24.79046058654785,
      "debug/policy_rejected_logps": -380.7741394042969,
      "debug/reference_chosen_logps": -393.5738525390625,
      "debug/reference_rejected_logps": -381.0361022949219,
      "epoch": 0.09375,
      "grad_norm": 5.494546355154901,
      "learning_rate": 1e-06,
      "logits/chosen": 27.064964294433594,
      "logits/rejected": 24.79046058654785,
      "logps/chosen": -393.77032470703125,
      "logps/rejected": -380.7741394042969,
      "loss": 0.4975,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0019646836444735527,
      "rewards/margins": -0.004584388807415962,
      "rewards/rejected": 0.0026197051629424095,
      "step": 3
    },
    {
      "debug/policy_chosen_logits": 28.537710189819336,
      "debug/policy_chosen_logps": -393.27496337890625,
      "debug/policy_rejected_logits": 26.111160278320312,
      "debug/policy_rejected_logps": -371.4323425292969,
      "debug/reference_chosen_logps": -394.78082275390625,
      "debug/reference_rejected_logps": -371.3717041015625,
      "epoch": 0.125,
      "grad_norm": 5.6289119594347605,
      "learning_rate": 1e-06,
      "logits/chosen": 28.537710189819336,
      "logits/rejected": 26.111160278320312,
      "logps/chosen": -393.27496337890625,
      "logps/rejected": -371.4323425292969,
      "loss": 0.496,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.015058821067214012,
      "rewards/margins": 0.015665167942643166,
      "rewards/rejected": -0.0006063459441065788,
      "step": 4
    },
    {
      "debug/policy_chosen_logits": 27.205202102661133,
      "debug/policy_chosen_logps": -372.9059753417969,
      "debug/policy_rejected_logits": 26.36362075805664,
      "debug/policy_rejected_logps": -375.8589172363281,
      "debug/reference_chosen_logps": -373.8547668457031,
      "debug/reference_rejected_logps": -376.77655029296875,
      "epoch": 0.15625,
      "grad_norm": 4.945940952646883,
      "learning_rate": 1e-06,
      "logits/chosen": 27.205202102661133,
      "logits/rejected": 26.36362075805664,
      "logps/chosen": -372.9059753417969,
      "logps/rejected": -375.8589172363281,
      "loss": 0.501,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009487838484346867,
      "rewards/margins": 0.0003112029517069459,
      "rewards/rejected": 0.00917663611471653,
      "step": 5
    },
    {
      "debug/policy_chosen_logits": 26.221603393554688,
      "debug/policy_chosen_logps": -389.5379638671875,
      "debug/policy_rejected_logits": 28.28006362915039,
      "debug/policy_rejected_logps": -388.9604187011719,
      "debug/reference_chosen_logps": -390.29022216796875,
      "debug/reference_rejected_logps": -389.8282775878906,
      "epoch": 0.1875,
      "grad_norm": 5.296809968342226,
      "learning_rate": 1e-06,
      "logits/chosen": 26.221603393554688,
      "logits/rejected": 28.28006362915039,
      "logps/chosen": -389.5379638671875,
      "logps/rejected": -388.9604187011719,
      "loss": 0.5017,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00752284936606884,
      "rewards/margins": -0.0011558537371456623,
      "rewards/rejected": 0.008678702637553215,
      "step": 6
    },
    {
      "debug/policy_chosen_logits": 26.66923713684082,
      "debug/policy_chosen_logps": -394.6316223144531,
      "debug/policy_rejected_logits": 25.254545211791992,
      "debug/policy_rejected_logps": -364.38299560546875,
      "debug/reference_chosen_logps": -395.24407958984375,
      "debug/reference_rejected_logps": -364.6360168457031,
      "epoch": 0.21875,
      "grad_norm": 5.136828721896042,
      "learning_rate": 1e-06,
      "logits/chosen": 26.66923713684082,
      "logits/rejected": 25.254545211791992,
      "logps/chosen": -394.6316223144531,
      "logps/rejected": -364.38299560546875,
      "loss": 0.4987,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006124534644186497,
      "rewards/margins": 0.0035945128183811903,
      "rewards/rejected": 0.0025300215929746628,
      "step": 7
    },
    {
      "debug/policy_chosen_logits": 23.028860092163086,
      "debug/policy_chosen_logps": -385.67840576171875,
      "debug/policy_rejected_logits": 25.49799919128418,
      "debug/policy_rejected_logps": -404.09539794921875,
      "debug/reference_chosen_logps": -386.231201171875,
      "debug/reference_rejected_logps": -403.6939392089844,
      "epoch": 0.25,
      "grad_norm": 5.138563586224073,
      "learning_rate": 1e-06,
      "logits/chosen": 23.028860092163086,
      "logits/rejected": 25.49799919128418,
      "logps/chosen": -385.67840576171875,
      "logps/rejected": -404.09539794921875,
      "loss": 0.4964,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005527915433049202,
      "rewards/margins": 0.009542694315314293,
      "rewards/rejected": -0.004014777950942516,
      "step": 8
    },
    {
      "debug/policy_chosen_logits": 27.734926223754883,
      "debug/policy_chosen_logps": -377.7676696777344,
      "debug/policy_rejected_logits": 26.249759674072266,
      "debug/policy_rejected_logps": -390.58367919921875,
      "debug/reference_chosen_logps": -377.9134216308594,
      "debug/reference_rejected_logps": -389.7505798339844,
      "epoch": 0.28125,
      "grad_norm": 5.126237673188204,
      "learning_rate": 1e-06,
      "logits/chosen": 27.734926223754883,
      "logits/rejected": 26.249759674072266,
      "logps/chosen": -377.7676696777344,
      "logps/rejected": -390.58367919921875,
      "loss": 0.4953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.001457519712857902,
      "rewards/margins": 0.009788626804947853,
      "rewards/rejected": -0.008331108838319778,
      "step": 9
    },
    {
      "debug/policy_chosen_logits": 27.97374725341797,
      "debug/policy_chosen_logps": -372.5227966308594,
      "debug/policy_rejected_logits": 26.68909454345703,
      "debug/policy_rejected_logps": -371.6484375,
      "debug/reference_chosen_logps": -371.6475830078125,
      "debug/reference_rejected_logps": -371.02789306640625,
      "epoch": 0.3125,
      "grad_norm": 5.826967853852101,
      "learning_rate": 1e-06,
      "logits/chosen": 27.97374725341797,
      "logits/rejected": 26.68909454345703,
      "logps/chosen": -372.5227966308594,
      "logps/rejected": -371.6484375,
      "loss": 0.5013,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008752173744142056,
      "rewards/margins": -0.002546653850004077,
      "rewards/rejected": -0.0062055205926299095,
      "step": 10
    },
    {
      "debug/policy_chosen_logits": 25.17486572265625,
      "debug/policy_chosen_logps": -387.5612487792969,
      "debug/policy_rejected_logits": 25.225900650024414,
      "debug/policy_rejected_logps": -394.29156494140625,
      "debug/reference_chosen_logps": -389.58349609375,
      "debug/reference_rejected_logps": -395.47021484375,
      "epoch": 0.34375,
      "grad_norm": 5.487120274982643,
      "learning_rate": 1e-06,
      "logits/chosen": 25.17486572265625,
      "logits/rejected": 25.225900650024414,
      "logps/chosen": -387.5612487792969,
      "logps/rejected": -394.29156494140625,
      "loss": 0.4966,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.020222166553139687,
      "rewards/margins": 0.008435897529125214,
      "rewards/rejected": 0.011786269955337048,
      "step": 11
    },
    {
      "debug/policy_chosen_logits": 25.62350082397461,
      "debug/policy_chosen_logps": -391.43536376953125,
      "debug/policy_rejected_logits": 26.19980239868164,
      "debug/policy_rejected_logps": -385.09454345703125,
      "debug/reference_chosen_logps": -391.71038818359375,
      "debug/reference_rejected_logps": -385.83221435546875,
      "epoch": 0.375,
      "grad_norm": 5.260475370442769,
      "learning_rate": 1e-06,
      "logits/chosen": 25.62350082397461,
      "logits/rejected": 26.19980239868164,
      "logps/chosen": -391.43536376953125,
      "logps/rejected": -385.09454345703125,
      "loss": 0.4959,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.0027502821758389473,
      "rewards/margins": -0.004626387730240822,
      "rewards/rejected": 0.007376670837402344,
      "step": 12
    },
    {
      "debug/policy_chosen_logits": 26.92571449279785,
      "debug/policy_chosen_logps": -368.7294006347656,
      "debug/policy_rejected_logits": 27.75008201599121,
      "debug/policy_rejected_logps": -406.4366149902344,
      "debug/reference_chosen_logps": -368.6622619628906,
      "debug/reference_rejected_logps": -406.2249450683594,
      "epoch": 0.40625,
      "grad_norm": 4.7848487788456415,
      "learning_rate": 1e-06,
      "logits/chosen": 26.92571449279785,
      "logits/rejected": 27.75008201599121,
      "logps/chosen": -368.7294006347656,
      "logps/rejected": -406.4366149902344,
      "loss": 0.4929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0006715008057653904,
      "rewards/margins": 0.0014451986644417048,
      "rewards/rejected": -0.002116698771715164,
      "step": 13
    },
    {
      "debug/policy_chosen_logits": 30.04401969909668,
      "debug/policy_chosen_logps": -375.1585998535156,
      "debug/policy_rejected_logits": 26.233510971069336,
      "debug/policy_rejected_logps": -391.10772705078125,
      "debug/reference_chosen_logps": -374.92877197265625,
      "debug/reference_rejected_logps": -390.80914306640625,
      "epoch": 0.4375,
      "grad_norm": 4.836397952988449,
      "learning_rate": 1e-06,
      "logits/chosen": 30.04401969909668,
      "logits/rejected": 26.233510971069336,
      "logps/chosen": -375.1585998535156,
      "logps/rejected": -391.10772705078125,
      "loss": 0.4978,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0022980873472988605,
      "rewards/margins": 0.0006880564615130424,
      "rewards/rejected": -0.0029861442744731903,
      "step": 14
    },
    {
      "debug/policy_chosen_logits": 26.359272003173828,
      "debug/policy_chosen_logps": -380.72393798828125,
      "debug/policy_rejected_logits": 25.454103469848633,
      "debug/policy_rejected_logps": -372.8114013671875,
      "debug/reference_chosen_logps": -381.0364990234375,
      "debug/reference_rejected_logps": -374.25555419921875,
      "epoch": 0.46875,
      "grad_norm": 5.802453188137246,
      "learning_rate": 1e-06,
      "logits/chosen": 26.359272003173828,
      "logits/rejected": 25.454103469848633,
      "logps/chosen": -380.72393798828125,
      "logps/rejected": -372.8114013671875,
      "loss": 0.5021,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0031254198402166367,
      "rewards/margins": -0.011316032148897648,
      "rewards/rejected": 0.01444145105779171,
      "step": 15
    },
    {
      "debug/policy_chosen_logits": 27.719757080078125,
      "debug/policy_chosen_logps": -387.31719970703125,
      "debug/policy_rejected_logits": 28.422988891601562,
      "debug/policy_rejected_logps": -394.9139404296875,
      "debug/reference_chosen_logps": -387.28955078125,
      "debug/reference_rejected_logps": -392.607666015625,
      "epoch": 0.5,
      "grad_norm": 5.284125486448873,
      "learning_rate": 1e-06,
      "logits/chosen": 27.719757080078125,
      "logits/rejected": 28.422988891601562,
      "logps/chosen": -387.31719970703125,
      "logps/rejected": -394.9139404296875,
      "loss": 0.4931,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.00027671828866004944,
      "rewards/margins": 0.022786367684602737,
      "rewards/rejected": -0.023063087835907936,
      "step": 16
    },
    {
      "debug/policy_chosen_logits": 26.237287521362305,
      "debug/policy_chosen_logps": -396.80377197265625,
      "debug/policy_rejected_logits": 26.251012802124023,
      "debug/policy_rejected_logps": -415.9134521484375,
      "debug/reference_chosen_logps": -396.8568115234375,
      "debug/reference_rejected_logps": -414.50372314453125,
      "epoch": 0.53125,
      "grad_norm": 5.422306895885996,
      "learning_rate": 1e-06,
      "logits/chosen": 26.237287521362305,
      "logits/rejected": 26.251012802124023,
      "logps/chosen": -396.80377197265625,
      "logps/rejected": -415.9134521484375,
      "loss": 0.4936,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0005303573561832309,
      "rewards/margins": 0.014627190306782722,
      "rewards/rejected": -0.014096831902861595,
      "step": 17
    },
    {
      "debug/policy_chosen_logits": 27.24745750427246,
      "debug/policy_chosen_logps": -400.6668701171875,
      "debug/policy_rejected_logits": 28.665220260620117,
      "debug/policy_rejected_logps": -404.18115234375,
      "debug/reference_chosen_logps": -400.91192626953125,
      "debug/reference_rejected_logps": -404.140625,
      "epoch": 0.5625,
      "grad_norm": 6.154342037709508,
      "learning_rate": 1e-06,
      "logits/chosen": 27.24745750427246,
      "logits/rejected": 28.665220260620117,
      "logps/chosen": -400.6668701171875,
      "logps/rejected": -404.18115234375,
      "loss": 0.504,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0024504470638930798,
      "rewards/margins": 0.002855682745575905,
      "rewards/rejected": -0.0004052352160215378,
      "step": 18
    },
    {
      "debug/policy_chosen_logits": 26.074438095092773,
      "debug/policy_chosen_logps": -368.0001220703125,
      "debug/policy_rejected_logits": 28.75902557373047,
      "debug/policy_rejected_logps": -402.38555908203125,
      "debug/reference_chosen_logps": -369.5516357421875,
      "debug/reference_rejected_logps": -402.15142822265625,
      "epoch": 0.59375,
      "grad_norm": 4.942495765333941,
      "learning_rate": 1e-06,
      "logits/chosen": 26.074438095092773,
      "logits/rejected": 28.75902557373047,
      "logps/chosen": -368.0001220703125,
      "logps/rejected": -402.38555908203125,
      "loss": 0.4971,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015514831990003586,
      "rewards/margins": 0.017856139689683914,
      "rewards/rejected": -0.002341308631002903,
      "step": 19
    },
    {
      "debug/policy_chosen_logits": 22.95450782775879,
      "debug/policy_chosen_logps": -387.22039794921875,
      "debug/policy_rejected_logits": 23.25604820251465,
      "debug/policy_rejected_logps": -407.40460205078125,
      "debug/reference_chosen_logps": -387.4991760253906,
      "debug/reference_rejected_logps": -406.79705810546875,
      "epoch": 0.625,
      "grad_norm": 5.841637379604238,
      "learning_rate": 1e-06,
      "logits/chosen": 22.95450782775879,
      "logits/rejected": 23.25604820251465,
      "logps/chosen": -387.22039794921875,
      "logps/rejected": -407.40460205078125,
      "loss": 0.4981,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0027874757070094347,
      "rewards/margins": 0.008862762711942196,
      "rewards/rejected": -0.00607528630644083,
      "step": 20
    },
    {
      "debug/policy_chosen_logits": 25.369287490844727,
      "debug/policy_chosen_logps": -384.18011474609375,
      "debug/policy_rejected_logits": 27.09587287902832,
      "debug/policy_rejected_logps": -374.6014709472656,
      "debug/reference_chosen_logps": -384.5386962890625,
      "debug/reference_rejected_logps": -374.75848388671875,
      "epoch": 0.65625,
      "grad_norm": 5.35593100802686,
      "learning_rate": 1e-06,
      "logits/chosen": 25.369287490844727,
      "logits/rejected": 27.09587287902832,
      "logps/chosen": -384.18011474609375,
      "logps/rejected": -374.6014709472656,
      "loss": 0.5014,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003585891332477331,
      "rewards/margins": 0.0020158004481345415,
      "rewards/rejected": 0.0015700910007581115,
      "step": 21
    },
    {
      "debug/policy_chosen_logits": 29.95963478088379,
      "debug/policy_chosen_logps": -412.9902648925781,
      "debug/policy_rejected_logits": 29.491188049316406,
      "debug/policy_rejected_logps": -388.817138671875,
      "debug/reference_chosen_logps": -413.6742248535156,
      "debug/reference_rejected_logps": -388.7025146484375,
      "epoch": 0.6875,
      "grad_norm": 5.750706618647552,
      "learning_rate": 1e-06,
      "logits/chosen": 29.95963478088379,
      "logits/rejected": 29.491188049316406,
      "logps/chosen": -412.9902648925781,
      "logps/rejected": -388.817138671875,
      "loss": 0.5016,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0068396758288145065,
      "rewards/margins": 0.007985686883330345,
      "rewards/rejected": -0.0011460117530077696,
      "step": 22
    },
    {
      "debug/policy_chosen_logits": 27.35663414001465,
      "debug/policy_chosen_logps": -381.54901123046875,
      "debug/policy_rejected_logits": 27.309879302978516,
      "debug/policy_rejected_logps": -408.41070556640625,
      "debug/reference_chosen_logps": -381.82025146484375,
      "debug/reference_rejected_logps": -406.42999267578125,
      "epoch": 0.71875,
      "grad_norm": 5.238622917805314,
      "learning_rate": 1e-06,
      "logits/chosen": 27.35663414001465,
      "logits/rejected": 27.309879302978516,
      "logps/chosen": -381.54901123046875,
      "logps/rejected": -408.41070556640625,
      "loss": 0.4947,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002712326357141137,
      "rewards/margins": 0.022518998011946678,
      "rewards/rejected": -0.019806671887636185,
      "step": 23
    },
    {
      "debug/policy_chosen_logits": 24.55692481994629,
      "debug/policy_chosen_logps": -373.54644775390625,
      "debug/policy_rejected_logits": 22.0833740234375,
      "debug/policy_rejected_logps": -383.18634033203125,
      "debug/reference_chosen_logps": -373.8763732910156,
      "debug/reference_rejected_logps": -384.98699951171875,
      "epoch": 0.75,
      "grad_norm": 5.752704029646181,
      "learning_rate": 1e-06,
      "logits/chosen": 24.55692481994629,
      "logits/rejected": 22.0833740234375,
      "logps/chosen": -373.54644775390625,
      "logps/rejected": -383.18634033203125,
      "loss": 0.5047,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00329933175817132,
      "rewards/margins": -0.014707411639392376,
      "rewards/rejected": 0.018006745725870132,
      "step": 24
    },
    {
      "debug/policy_chosen_logits": 24.850391387939453,
      "debug/policy_chosen_logps": -386.2474365234375,
      "debug/policy_rejected_logits": 29.567493438720703,
      "debug/policy_rejected_logps": -422.58990478515625,
      "debug/reference_chosen_logps": -386.5673828125,
      "debug/reference_rejected_logps": -422.57989501953125,
      "epoch": 0.78125,
      "grad_norm": 5.457291481304808,
      "learning_rate": 1e-06,
      "logits/chosen": 24.850391387939453,
      "logits/rejected": 29.567493438720703,
      "logps/chosen": -386.2474365234375,
      "logps/rejected": -422.58990478515625,
      "loss": 0.4937,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.003199271857738495,
      "rewards/margins": 0.0032989501487463713,
      "rewards/rejected": -9.96782910078764e-05,
      "step": 25
    },
    {
      "debug/policy_chosen_logits": 24.353761672973633,
      "debug/policy_chosen_logps": -365.70977783203125,
      "debug/policy_rejected_logits": 27.883697509765625,
      "debug/policy_rejected_logps": -387.9311828613281,
      "debug/reference_chosen_logps": -365.4722900390625,
      "debug/reference_rejected_logps": -386.3861083984375,
      "epoch": 0.8125,
      "grad_norm": 6.268906937225549,
      "learning_rate": 1e-06,
      "logits/chosen": 24.353761672973633,
      "logits/rejected": 27.883697509765625,
      "logps/chosen": -365.70977783203125,
      "logps/rejected": -387.9311828613281,
      "loss": 0.492,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0023751831613481045,
      "rewards/margins": 0.013075370341539383,
      "rewards/rejected": -0.0154505530372262,
      "step": 26
    },
    {
      "debug/policy_chosen_logits": 29.132570266723633,
      "debug/policy_chosen_logps": -383.54681396484375,
      "debug/policy_rejected_logits": 30.820039749145508,
      "debug/policy_rejected_logps": -391.74847412109375,
      "debug/reference_chosen_logps": -384.86737060546875,
      "debug/reference_rejected_logps": -390.72613525390625,
      "epoch": 0.84375,
      "grad_norm": 5.319109456599871,
      "learning_rate": 1e-06,
      "logits/chosen": 29.132570266723633,
      "logits/rejected": 30.820039749145508,
      "logps/chosen": -383.54681396484375,
      "logps/rejected": -391.74847412109375,
      "loss": 0.4953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013205718249082565,
      "rewards/margins": 0.023428915068507195,
      "rewards/rejected": -0.010223197750747204,
      "step": 27
    },
    {
      "debug/policy_chosen_logits": 21.42544174194336,
      "debug/policy_chosen_logps": -359.3302917480469,
      "debug/policy_rejected_logits": 23.681625366210938,
      "debug/policy_rejected_logps": -420.1634826660156,
      "debug/reference_chosen_logps": -360.5409240722656,
      "debug/reference_rejected_logps": -418.49676513671875,
      "epoch": 0.875,
      "grad_norm": 5.766637040961972,
      "learning_rate": 1e-06,
      "logits/chosen": 21.42544174194336,
      "logits/rejected": 23.681625366210938,
      "logps/chosen": -359.3302917480469,
      "logps/rejected": -420.1634826660156,
      "loss": 0.4952,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012106247246265411,
      "rewards/margins": 0.028773421421647072,
      "rewards/rejected": -0.01666717603802681,
      "step": 28
    },
    {
      "debug/policy_chosen_logits": 23.576292037963867,
      "debug/policy_chosen_logps": -389.7811279296875,
      "debug/policy_rejected_logits": 25.178329467773438,
      "debug/policy_rejected_logps": -380.7835998535156,
      "debug/reference_chosen_logps": -388.56658935546875,
      "debug/reference_rejected_logps": -380.28936767578125,
      "epoch": 0.90625,
      "grad_norm": 5.149123243810572,
      "learning_rate": 1e-06,
      "logits/chosen": 23.576292037963867,
      "logits/rejected": 25.178329467773438,
      "logps/chosen": -389.7811279296875,
      "logps/rejected": -380.7835998535156,
      "loss": 0.5005,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01214546151459217,
      "rewards/margins": -0.007203062996268272,
      "rewards/rejected": -0.004942398518323898,
      "step": 29
    },
    {
      "debug/policy_chosen_logits": 25.0103702545166,
      "debug/policy_chosen_logps": -374.01544189453125,
      "debug/policy_rejected_logits": 24.813074111938477,
      "debug/policy_rejected_logps": -402.4690246582031,
      "debug/reference_chosen_logps": -373.6084899902344,
      "debug/reference_rejected_logps": -400.3970947265625,
      "epoch": 0.9375,
      "grad_norm": 5.054574585698812,
      "learning_rate": 1e-06,
      "logits/chosen": 25.0103702545166,
      "logits/rejected": 24.813074111938477,
      "logps/chosen": -374.01544189453125,
      "logps/rejected": -402.4690246582031,
      "loss": 0.4868,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.00406951829791069,
      "rewards/margins": 0.016650084406137466,
      "rewards/rejected": -0.020719602704048157,
      "step": 30
    },
    {
      "debug/policy_chosen_logits": 24.653635025024414,
      "debug/policy_chosen_logps": -389.5201721191406,
      "debug/policy_rejected_logits": 26.097084045410156,
      "debug/policy_rejected_logps": -385.3985290527344,
      "debug/reference_chosen_logps": -388.95758056640625,
      "debug/reference_rejected_logps": -384.1873779296875,
      "epoch": 0.96875,
      "grad_norm": 5.42566873511884,
      "learning_rate": 1e-06,
      "logits/chosen": 24.653635025024414,
      "logits/rejected": 26.097084045410156,
      "logps/chosen": -389.5201721191406,
      "logps/rejected": -385.3985290527344,
      "loss": 0.4971,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005625876598060131,
      "rewards/margins": 0.006485518999397755,
      "rewards/rejected": -0.012111397460103035,
      "step": 31
    },
    {
      "debug/policy_chosen_logits": 26.803712844848633,
      "debug/policy_chosen_logps": -409.23480224609375,
      "debug/policy_rejected_logits": 27.91266632080078,
      "debug/policy_rejected_logps": -441.89007568359375,
      "debug/reference_chosen_logps": -409.35791015625,
      "debug/reference_rejected_logps": -439.5430908203125,
      "epoch": 1.0,
      "grad_norm": 5.122199540354154,
      "learning_rate": 1e-06,
      "logits/chosen": 26.803712844848633,
      "logits/rejected": 27.91266632080078,
      "logps/chosen": -409.23480224609375,
      "logps/rejected": -441.89007568359375,
      "loss": 0.4517,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0012308494187891483,
      "rewards/margins": 0.024701077491044998,
      "rewards/rejected": -0.023470228537917137,
      "step": 32
    },
    {
      "epoch": 1.0,
      "step": 32,
      "total_flos": 0.0,
      "train_loss": 0.49607059359550476,
      "train_runtime": 386.5625,
      "train_samples_per_second": 5.161,
      "train_steps_per_second": 0.083
    }
  ],
  "logging_steps": 1,
  "max_steps": 32,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}