{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 33,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "debug/policy_chosen_logits": 26.247488021850586,
      "debug/policy_chosen_logps": -381.44134521484375,
      "debug/policy_rejected_logits": 26.089710235595703,
      "debug/policy_rejected_logps": -404.22979736328125,
      "debug/reference_chosen_logps": -381.44134521484375,
      "debug/reference_rejected_logps": -404.22979736328125,
      "epoch": 0.030303030303030304,
      "grad_norm": 5.442552264842066,
      "learning_rate": 1e-06,
      "logits/chosen": 26.247488021850586,
      "logits/rejected": 26.089710235595703,
      "logps/chosen": -381.44134521484375,
      "logps/rejected": -404.22979736328125,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "debug/policy_chosen_logits": 29.335710525512695,
      "debug/policy_chosen_logps": -403.6748352050781,
      "debug/policy_rejected_logits": 29.501441955566406,
      "debug/policy_rejected_logps": -394.9062194824219,
      "debug/reference_chosen_logps": -403.30743408203125,
      "debug/reference_rejected_logps": -394.7815856933594,
      "epoch": 0.06060606060606061,
      "grad_norm": 5.814697031201472,
      "learning_rate": 1e-06,
      "logits/chosen": 29.335710525512695,
      "logits/rejected": 29.501441955566406,
      "logps/chosen": -403.6748352050781,
      "logps/rejected": -394.9062194824219,
      "loss": 0.5004,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0036742021329700947,
      "rewards/margins": -0.0024280548095703125,
      "rewards/rejected": -0.0012461473233997822,
      "step": 2
    },
    {
      "debug/policy_chosen_logits": 26.286334991455078,
      "debug/policy_chosen_logps": -385.650634765625,
      "debug/policy_rejected_logits": 26.62445640563965,
      "debug/policy_rejected_logps": -411.38519287109375,
      "debug/reference_chosen_logps": -385.63262939453125,
      "debug/reference_rejected_logps": -411.11822509765625,
      "epoch": 0.09090909090909091,
      "grad_norm": 5.064121289594221,
      "learning_rate": 1e-06,
      "logits/chosen": 26.286334991455078,
      "logits/rejected": 26.62445640563965,
      "logps/chosen": -385.650634765625,
      "logps/rejected": -411.38519287109375,
      "loss": 0.5017,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.00018005399033427238,
      "rewards/margins": 0.0024895858950912952,
      "rewards/rejected": -0.0026696398854255676,
      "step": 3
    },
    {
      "debug/policy_chosen_logits": 30.732887268066406,
      "debug/policy_chosen_logps": -401.61358642578125,
      "debug/policy_rejected_logits": 26.973230361938477,
      "debug/policy_rejected_logps": -379.72772216796875,
      "debug/reference_chosen_logps": -401.27716064453125,
      "debug/reference_rejected_logps": -380.01727294921875,
      "epoch": 0.12121212121212122,
      "grad_norm": 4.93218202903919,
      "learning_rate": 1e-06,
      "logits/chosen": 30.732887268066406,
      "logits/rejected": 26.973230361938477,
      "logps/chosen": -401.61358642578125,
      "logps/rejected": -379.72772216796875,
      "loss": 0.5001,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0033642961643636227,
      "rewards/margins": -0.00626018550246954,
      "rewards/rejected": 0.002895889338105917,
      "step": 4
    },
    {
      "debug/policy_chosen_logits": 26.53725814819336,
      "debug/policy_chosen_logps": -364.45782470703125,
      "debug/policy_rejected_logits": 29.11540412902832,
      "debug/policy_rejected_logps": -386.5697326660156,
      "debug/reference_chosen_logps": -364.01983642578125,
      "debug/reference_rejected_logps": -386.7102966308594,
      "epoch": 0.15151515151515152,
      "grad_norm": 4.732508848857073,
      "learning_rate": 1e-06,
      "logits/chosen": 26.53725814819336,
      "logits/rejected": 29.11540412902832,
      "logps/chosen": -364.45782470703125,
      "logps/rejected": -386.5697326660156,
      "loss": 0.5019,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0043797302059829235,
      "rewards/margins": -0.005785560701042414,
      "rewards/rejected": 0.0014058304950594902,
      "step": 5
    },
    {
      "debug/policy_chosen_logits": 27.333480834960938,
      "debug/policy_chosen_logps": -371.4402770996094,
      "debug/policy_rejected_logits": 28.14923858642578,
      "debug/policy_rejected_logps": -380.748291015625,
      "debug/reference_chosen_logps": -370.5325927734375,
      "debug/reference_rejected_logps": -379.92169189453125,
      "epoch": 0.18181818181818182,
      "grad_norm": 5.12780395559187,
      "learning_rate": 1e-06,
      "logits/chosen": 27.333480834960938,
      "logits/rejected": 28.14923858642578,
      "logps/chosen": -371.4402770996094,
      "logps/rejected": -380.748291015625,
      "loss": 0.5005,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009076766669750214,
      "rewards/margins": -0.0008102413266897202,
      "rewards/rejected": -0.008266524411737919,
      "step": 6
    },
    {
      "debug/policy_chosen_logits": 27.809894561767578,
      "debug/policy_chosen_logps": -409.5631408691406,
      "debug/policy_rejected_logits": 25.189990997314453,
      "debug/policy_rejected_logps": -424.32916259765625,
      "debug/reference_chosen_logps": -409.25799560546875,
      "debug/reference_rejected_logps": -424.49981689453125,
      "epoch": 0.21212121212121213,
      "grad_norm": 6.280279145214255,
      "learning_rate": 1e-06,
      "logits/chosen": 27.809894561767578,
      "logits/rejected": 25.189990997314453,
      "logps/chosen": -409.5631408691406,
      "logps/rejected": -424.32916259765625,
      "loss": 0.4977,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0030513377860188484,
      "rewards/margins": -0.0047579575330019,
      "rewards/rejected": 0.0017066190484911203,
      "step": 7
    },
    {
      "debug/policy_chosen_logits": 28.680850982666016,
      "debug/policy_chosen_logps": -367.8634033203125,
      "debug/policy_rejected_logits": 26.834354400634766,
      "debug/policy_rejected_logps": -373.39215087890625,
      "debug/reference_chosen_logps": -367.77484130859375,
      "debug/reference_rejected_logps": -373.0135192871094,
      "epoch": 0.24242424242424243,
      "grad_norm": 6.296849117602656,
      "learning_rate": 1e-06,
      "logits/chosen": 28.680850982666016,
      "logits/rejected": 26.834354400634766,
      "logps/chosen": -367.8634033203125,
      "logps/rejected": -373.39215087890625,
      "loss": 0.5004,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.000885658198967576,
      "rewards/margins": 0.002900313585996628,
      "rewards/rejected": -0.003785972483456135,
      "step": 8
    },
    {
      "debug/policy_chosen_logits": 26.558706283569336,
      "debug/policy_chosen_logps": -407.4287414550781,
      "debug/policy_rejected_logits": 27.462989807128906,
      "debug/policy_rejected_logps": -404.10052490234375,
      "debug/reference_chosen_logps": -407.3221130371094,
      "debug/reference_rejected_logps": -403.45672607421875,
      "epoch": 0.2727272727272727,
      "grad_norm": 5.217727377560115,
      "learning_rate": 1e-06,
      "logits/chosen": 26.558706283569336,
      "logits/rejected": 27.462989807128906,
      "logps/chosen": -407.4287414550781,
      "logps/rejected": -404.10052490234375,
      "loss": 0.4967,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0010662460699677467,
      "rewards/margins": 0.0053720856085419655,
      "rewards/rejected": -0.006438331678509712,
      "step": 9
    },
    {
      "debug/policy_chosen_logits": 27.330812454223633,
      "debug/policy_chosen_logps": -397.4725646972656,
      "debug/policy_rejected_logits": 27.048776626586914,
      "debug/policy_rejected_logps": -385.05328369140625,
      "debug/reference_chosen_logps": -397.6583251953125,
      "debug/reference_rejected_logps": -385.533203125,
      "epoch": 0.30303030303030304,
      "grad_norm": 5.264064218965023,
      "learning_rate": 1e-06,
      "logits/chosen": 27.330812454223633,
      "logits/rejected": 27.048776626586914,
      "logps/chosen": -397.4725646972656,
      "logps/rejected": -385.05328369140625,
      "loss": 0.4993,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0018573375418782234,
      "rewards/margins": -0.002941780723631382,
      "rewards/rejected": 0.004799118265509605,
      "step": 10
    },
    {
      "debug/policy_chosen_logits": 26.38730812072754,
      "debug/policy_chosen_logps": -394.4657287597656,
      "debug/policy_rejected_logits": 25.263452529907227,
      "debug/policy_rejected_logps": -401.41778564453125,
      "debug/reference_chosen_logps": -394.7672119140625,
      "debug/reference_rejected_logps": -401.0439453125,
      "epoch": 0.3333333333333333,
      "grad_norm": 5.091598075709861,
      "learning_rate": 1e-06,
      "logits/chosen": 26.38730812072754,
      "logits/rejected": 25.263452529907227,
      "logps/chosen": -394.4657287597656,
      "logps/rejected": -401.41778564453125,
      "loss": 0.4959,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0030147552024573088,
      "rewards/margins": 0.006753234658390284,
      "rewards/rejected": -0.003738479223102331,
      "step": 11
    },
    {
      "debug/policy_chosen_logits": 29.810720443725586,
      "debug/policy_chosen_logps": -365.6493225097656,
      "debug/policy_rejected_logits": 29.309846878051758,
      "debug/policy_rejected_logps": -382.3474426269531,
      "debug/reference_chosen_logps": -366.44732666015625,
      "debug/reference_rejected_logps": -382.22650146484375,
      "epoch": 0.36363636363636365,
      "grad_norm": 4.68129717907089,
      "learning_rate": 1e-06,
      "logits/chosen": 29.810720443725586,
      "logits/rejected": 29.309846878051758,
      "logps/chosen": -365.6493225097656,
      "logps/rejected": -382.3474426269531,
      "loss": 0.4974,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.00797988846898079,
      "rewards/margins": 0.009189032949507236,
      "rewards/rejected": -0.0012091444805264473,
      "step": 12
    },
    {
      "debug/policy_chosen_logits": 25.008668899536133,
      "debug/policy_chosen_logps": -400.12109375,
      "debug/policy_rejected_logits": 23.562170028686523,
      "debug/policy_rejected_logps": -360.582275390625,
      "debug/reference_chosen_logps": -400.1289978027344,
      "debug/reference_rejected_logps": -360.8138427734375,
      "epoch": 0.3939393939393939,
      "grad_norm": 4.901760910772593,
      "learning_rate": 1e-06,
      "logits/chosen": 25.008668899536133,
      "logits/rejected": 23.562170028686523,
      "logps/chosen": -400.12109375,
      "logps/rejected": -360.582275390625,
      "loss": 0.5,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 7.907883264124393e-05,
      "rewards/margins": -0.0022365953773260117,
      "rewards/rejected": 0.002315673977136612,
      "step": 13
    },
    {
      "debug/policy_chosen_logits": 28.53821563720703,
      "debug/policy_chosen_logps": -389.4108581542969,
      "debug/policy_rejected_logits": 26.43337631225586,
      "debug/policy_rejected_logps": -399.672607421875,
      "debug/reference_chosen_logps": -389.0559387207031,
      "debug/reference_rejected_logps": -400.2510986328125,
      "epoch": 0.42424242424242425,
      "grad_norm": 5.316323576734348,
      "learning_rate": 1e-06,
      "logits/chosen": 28.53821563720703,
      "logits/rejected": 26.43337631225586,
      "logps/chosen": -389.4108581542969,
      "logps/rejected": -399.672607421875,
      "loss": 0.4975,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0035491182934492826,
      "rewards/margins": -0.009334182366728783,
      "rewards/rejected": 0.005785064306110144,
      "step": 14
    },
    {
      "debug/policy_chosen_logits": 29.288440704345703,
      "debug/policy_chosen_logps": -396.9311218261719,
      "debug/policy_rejected_logits": 25.288602828979492,
      "debug/policy_rejected_logps": -369.3037109375,
      "debug/reference_chosen_logps": -397.51153564453125,
      "debug/reference_rejected_logps": -369.73687744140625,
      "epoch": 0.45454545454545453,
      "grad_norm": 4.921365130482872,
      "learning_rate": 1e-06,
      "logits/chosen": 29.288440704345703,
      "logits/rejected": 25.288602828979492,
      "logps/chosen": -396.9311218261719,
      "logps/rejected": -369.3037109375,
      "loss": 0.4949,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.005803985986858606,
      "rewards/margins": 0.0014723972417414188,
      "rewards/rejected": 0.0043315887451171875,
      "step": 15
    },
    {
      "debug/policy_chosen_logits": 25.585439682006836,
      "debug/policy_chosen_logps": -410.58551025390625,
      "debug/policy_rejected_logits": 24.364900588989258,
      "debug/policy_rejected_logps": -394.73638916015625,
      "debug/reference_chosen_logps": -411.3353271484375,
      "debug/reference_rejected_logps": -394.3494873046875,
      "epoch": 0.48484848484848486,
      "grad_norm": 5.084152192534468,
      "learning_rate": 1e-06,
      "logits/chosen": 25.585439682006836,
      "logits/rejected": 24.364900588989258,
      "logps/chosen": -410.58551025390625,
      "logps/rejected": -394.73638916015625,
      "loss": 0.4953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0074979402124881744,
      "rewards/margins": 0.01136699691414833,
      "rewards/rejected": -0.0038690564688295126,
      "step": 16
    },
    {
      "debug/policy_chosen_logits": 25.545652389526367,
      "debug/policy_chosen_logps": -390.6317138671875,
      "debug/policy_rejected_logits": 24.665712356567383,
      "debug/policy_rejected_logps": -388.66748046875,
      "debug/reference_chosen_logps": -392.0401611328125,
      "debug/reference_rejected_logps": -388.7672119140625,
      "epoch": 0.5151515151515151,
      "grad_norm": 6.040733481743162,
      "learning_rate": 1e-06,
      "logits/chosen": 25.545652389526367,
      "logits/rejected": 24.665712356567383,
      "logps/chosen": -390.6317138671875,
      "logps/rejected": -388.66748046875,
      "loss": 0.4979,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01408412866294384,
      "rewards/margins": 0.013087119907140732,
      "rewards/rejected": 0.0009970092214643955,
      "step": 17
    },
    {
      "debug/policy_chosen_logits": 21.49858283996582,
      "debug/policy_chosen_logps": -373.1883850097656,
      "debug/policy_rejected_logits": 20.456966400146484,
      "debug/policy_rejected_logps": -389.541015625,
      "debug/reference_chosen_logps": -374.1577453613281,
      "debug/reference_rejected_logps": -389.02264404296875,
      "epoch": 0.5454545454545454,
      "grad_norm": 4.954322411747276,
      "learning_rate": 1e-06,
      "logits/chosen": 21.49858283996582,
      "logits/rejected": 20.456966400146484,
      "logps/chosen": -373.1883850097656,
      "logps/rejected": -389.541015625,
      "loss": 0.4967,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.009693756699562073,
      "rewards/margins": 0.01487777754664421,
      "rewards/rejected": -0.005184021312743425,
      "step": 18
    },
    {
      "debug/policy_chosen_logits": 28.040218353271484,
      "debug/policy_chosen_logps": -395.2466125488281,
      "debug/policy_rejected_logits": 30.119606018066406,
      "debug/policy_rejected_logps": -410.98419189453125,
      "debug/reference_chosen_logps": -394.3302001953125,
      "debug/reference_rejected_logps": -410.0677490234375,
      "epoch": 0.5757575757575758,
      "grad_norm": 4.970008551225456,
      "learning_rate": 1e-06,
      "logits/chosen": 28.040218353271484,
      "logits/rejected": 30.119606018066406,
      "logps/chosen": -395.2466125488281,
      "logps/rejected": -410.98419189453125,
      "loss": 0.5008,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009163931012153625,
      "rewards/margins": 5.345791578292847e-07,
      "rewards/rejected": -0.009164467453956604,
      "step": 19
    },
    {
      "debug/policy_chosen_logits": 25.36637306213379,
      "debug/policy_chosen_logps": -359.6551208496094,
      "debug/policy_rejected_logits": 28.496570587158203,
      "debug/policy_rejected_logps": -396.62646484375,
      "debug/reference_chosen_logps": -359.1936950683594,
      "debug/reference_rejected_logps": -394.8607177734375,
      "epoch": 0.6060606060606061,
      "grad_norm": 5.36738350658573,
      "learning_rate": 1e-06,
      "logits/chosen": 25.36637306213379,
      "logits/rejected": 28.496570587158203,
      "logps/chosen": -359.6551208496094,
      "logps/rejected": -396.62646484375,
      "loss": 0.493,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00461437227204442,
      "rewards/margins": 0.013043059967458248,
      "rewards/rejected": -0.017657432705163956,
      "step": 20
    },
    {
      "debug/policy_chosen_logits": 27.611148834228516,
      "debug/policy_chosen_logps": -388.72259521484375,
      "debug/policy_rejected_logits": 27.41425323486328,
      "debug/policy_rejected_logps": -376.91302490234375,
      "debug/reference_chosen_logps": -388.17486572265625,
      "debug/reference_rejected_logps": -376.16900634765625,
      "epoch": 0.6363636363636364,
      "grad_norm": 4.90749395193054,
      "learning_rate": 1e-06,
      "logits/chosen": 27.611148834228516,
      "logits/rejected": 27.41425323486328,
      "logps/chosen": -388.72259521484375,
      "logps/rejected": -376.91302490234375,
      "loss": 0.4944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00547714252024889,
      "rewards/margins": 0.0019630808383226395,
      "rewards/rejected": -0.007440223358571529,
      "step": 21
    },
    {
      "debug/policy_chosen_logits": 31.425703048706055,
      "debug/policy_chosen_logps": -415.8355712890625,
      "debug/policy_rejected_logits": 30.123226165771484,
      "debug/policy_rejected_logps": -390.62847900390625,
      "debug/reference_chosen_logps": -416.2061767578125,
      "debug/reference_rejected_logps": -391.3880310058594,
      "epoch": 0.6666666666666666,
      "grad_norm": 5.726245430763464,
      "learning_rate": 1e-06,
      "logits/chosen": 31.425703048706055,
      "logits/rejected": 30.123226165771484,
      "logps/chosen": -415.8355712890625,
      "logps/rejected": -390.62847900390625,
      "loss": 0.491,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003706092946231365,
      "rewards/margins": -0.003889350686222315,
      "rewards/rejected": 0.00759544363245368,
      "step": 22
    },
    {
      "debug/policy_chosen_logits": 28.34208869934082,
      "debug/policy_chosen_logps": -383.0400085449219,
      "debug/policy_rejected_logits": 27.740306854248047,
      "debug/policy_rejected_logps": -396.38739013671875,
      "debug/reference_chosen_logps": -382.1289367675781,
      "debug/reference_rejected_logps": -395.863525390625,
      "epoch": 0.696969696969697,
      "grad_norm": 6.5017943561734475,
      "learning_rate": 1e-06,
      "logits/chosen": 28.34208869934082,
      "logits/rejected": 27.740306854248047,
      "logps/chosen": -383.0400085449219,
      "logps/rejected": -396.38739013671875,
      "loss": 0.4974,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009110527113080025,
      "rewards/margins": -0.003872184082865715,
      "rewards/rejected": -0.005238342098891735,
      "step": 23
    },
    {
      "debug/policy_chosen_logits": 28.894771575927734,
      "debug/policy_chosen_logps": -379.154296875,
      "debug/policy_rejected_logits": 29.083221435546875,
      "debug/policy_rejected_logps": -398.38275146484375,
      "debug/reference_chosen_logps": -379.416015625,
      "debug/reference_rejected_logps": -397.15728759765625,
      "epoch": 0.7272727272727273,
      "grad_norm": 5.2990691392738665,
      "learning_rate": 1e-06,
      "logits/chosen": 28.894771575927734,
      "logits/rejected": 29.083221435546875,
      "logps/chosen": -379.154296875,
      "logps/rejected": -398.38275146484375,
      "loss": 0.4945,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0026170345954596996,
      "rewards/margins": 0.014871787279844284,
      "rewards/rejected": -0.012254752218723297,
      "step": 24
    },
    {
      "debug/policy_chosen_logits": 27.720739364624023,
      "debug/policy_chosen_logps": -386.32635498046875,
      "debug/policy_rejected_logits": 24.095592498779297,
      "debug/policy_rejected_logps": -369.91998291015625,
      "debug/reference_chosen_logps": -385.1112365722656,
      "debug/reference_rejected_logps": -369.97222900390625,
      "epoch": 0.7575757575757576,
      "grad_norm": 5.341763911189841,
      "learning_rate": 1e-06,
      "logits/chosen": 27.720739364624023,
      "logits/rejected": 24.095592498779297,
      "logps/chosen": -386.32635498046875,
      "logps/rejected": -369.91998291015625,
      "loss": 0.4978,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.0121515654027462,
      "rewards/margins": -0.012673910707235336,
      "rewards/rejected": 0.0005223462358117104,
      "step": 25
    },
    {
      "debug/policy_chosen_logits": 26.473604202270508,
      "debug/policy_chosen_logps": -406.0578308105469,
      "debug/policy_rejected_logits": 25.949445724487305,
      "debug/policy_rejected_logps": -396.72174072265625,
      "debug/reference_chosen_logps": -406.4796447753906,
      "debug/reference_rejected_logps": -396.7256774902344,
      "epoch": 0.7878787878787878,
      "grad_norm": 5.2984045842092815,
      "learning_rate": 1e-06,
      "logits/chosen": 26.473604202270508,
      "logits/rejected": 25.949445724487305,
      "logps/chosen": -406.0578308105469,
      "logps/rejected": -396.72174072265625,
      "loss": 0.5009,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004218368325382471,
      "rewards/margins": 0.0041790008544921875,
      "rewards/rejected": 3.936793655157089e-05,
      "step": 26
    },
    {
      "debug/policy_chosen_logits": 25.699703216552734,
      "debug/policy_chosen_logps": -412.0224609375,
      "debug/policy_rejected_logits": 22.656808853149414,
      "debug/policy_rejected_logps": -371.5995178222656,
      "debug/reference_chosen_logps": -411.7939453125,
      "debug/reference_rejected_logps": -374.21875,
      "epoch": 0.8181818181818182,
      "grad_norm": 5.565761201808661,
      "learning_rate": 1e-06,
      "logits/chosen": 25.699703216552734,
      "logits/rejected": 22.656808853149414,
      "logps/chosen": -412.0224609375,
      "logps/rejected": -371.5995178222656,
      "loss": 0.5049,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.0022848891094326973,
      "rewards/margins": -0.028477095067501068,
      "rewards/rejected": 0.026192206889390945,
      "step": 27
    },
    {
      "debug/policy_chosen_logits": 28.45078468322754,
      "debug/policy_chosen_logps": -388.2000732421875,
      "debug/policy_rejected_logits": 24.951025009155273,
      "debug/policy_rejected_logps": -383.76446533203125,
      "debug/reference_chosen_logps": -389.6691589355469,
      "debug/reference_rejected_logps": -383.29156494140625,
      "epoch": 0.8484848484848485,
      "grad_norm": 5.763656144736158,
      "learning_rate": 1e-06,
      "logits/chosen": 28.45078468322754,
      "logits/rejected": 24.951025009155273,
      "logps/chosen": -388.2000732421875,
      "logps/rejected": -383.76446533203125,
      "loss": 0.4931,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014691008254885674,
      "rewards/margins": 0.019420089200139046,
      "rewards/rejected": -0.004729080013930798,
      "step": 28
    },
    {
      "debug/policy_chosen_logits": 27.9750919342041,
      "debug/policy_chosen_logps": -392.9916076660156,
      "debug/policy_rejected_logits": 30.114521026611328,
      "debug/policy_rejected_logps": -411.13897705078125,
      "debug/reference_chosen_logps": -394.3369140625,
      "debug/reference_rejected_logps": -412.72320556640625,
      "epoch": 0.8787878787878788,
      "grad_norm": 5.254216092353236,
      "learning_rate": 1e-06,
      "logits/chosen": 27.9750919342041,
      "logits/rejected": 30.114521026611328,
      "logps/chosen": -392.9916076660156,
      "logps/rejected": -411.13897705078125,
      "loss": 0.4989,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.013453138992190361,
      "rewards/margins": -0.0023886507842689753,
      "rewards/rejected": 0.015841789543628693,
      "step": 29
    },
    {
      "debug/policy_chosen_logits": 29.86224937438965,
      "debug/policy_chosen_logps": -391.58087158203125,
      "debug/policy_rejected_logits": 30.319177627563477,
      "debug/policy_rejected_logps": -398.785400390625,
      "debug/reference_chosen_logps": -391.1322326660156,
      "debug/reference_rejected_logps": -397.64776611328125,
      "epoch": 0.9090909090909091,
      "grad_norm": 5.01142693933464,
      "learning_rate": 1e-06,
      "logits/chosen": 29.86224937438965,
      "logits/rejected": 30.319177627563477,
      "logps/chosen": -391.58087158203125,
      "logps/rejected": -398.785400390625,
      "loss": 0.4928,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0044861603528261185,
      "rewards/margins": 0.006889915093779564,
      "rewards/rejected": -0.011376074515283108,
      "step": 30
    },
    {
      "debug/policy_chosen_logits": 24.41091537475586,
      "debug/policy_chosen_logps": -374.98590087890625,
      "debug/policy_rejected_logits": 25.041379928588867,
      "debug/policy_rejected_logps": -392.67889404296875,
      "debug/reference_chosen_logps": -376.893798828125,
      "debug/reference_rejected_logps": -391.7608337402344,
      "epoch": 0.9393939393939394,
      "grad_norm": 5.68017363572054,
      "learning_rate": 1e-06,
      "logits/chosen": 24.41091537475586,
      "logits/rejected": 25.041379928588867,
      "logps/chosen": -374.98590087890625,
      "logps/rejected": -392.67889404296875,
      "loss": 0.4959,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.019078826531767845,
      "rewards/margins": 0.028259430080652237,
      "rewards/rejected": -0.009180602617561817,
      "step": 31
    },
    {
      "debug/policy_chosen_logits": 31.064552307128906,
      "debug/policy_chosen_logps": -416.615966796875,
      "debug/policy_rejected_logits": 27.05549430847168,
      "debug/policy_rejected_logps": -407.7713623046875,
      "debug/reference_chosen_logps": -416.4413757324219,
      "debug/reference_rejected_logps": -407.89178466796875,
      "epoch": 0.9696969696969697,
      "grad_norm": 5.3155545548133,
      "learning_rate": 1e-06,
      "logits/chosen": 31.064552307128906,
      "logits/rejected": 27.05549430847168,
      "logps/chosen": -416.615966796875,
      "logps/rejected": -407.7713623046875,
      "loss": 0.4929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.001745910383760929,
      "rewards/margins": -0.002950286725535989,
      "rewards/rejected": 0.0012043765746057034,
      "step": 32
    },
    {
      "debug/policy_chosen_logits": 28.52979850769043,
      "debug/policy_chosen_logps": -404.65618896484375,
      "debug/policy_rejected_logits": 23.76580047607422,
      "debug/policy_rejected_logps": -377.091796875,
      "debug/reference_chosen_logps": -406.3551940917969,
      "debug/reference_rejected_logps": -375.24481201171875,
      "epoch": 1.0,
      "grad_norm": 4.941707849448349,
      "learning_rate": 1e-06,
      "logits/chosen": 28.52979850769043,
      "logits/rejected": 23.76580047607422,
      "logps/chosen": -404.65618896484375,
      "logps/rejected": -377.091796875,
      "loss": 0.4529,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.016990013420581818,
      "rewards/margins": 0.03545970842242241,
      "rewards/rejected": -0.01846969500184059,
      "step": 33
    },
    {
      "epoch": 1.0,
      "step": 33,
      "total_flos": 0.0,
      "train_loss": 0.4962217591025613,
      "train_runtime": 388.3481,
      "train_samples_per_second": 5.302,
      "train_steps_per_second": 0.085
    }
  ],
  "logging_steps": 1,
  "max_steps": 33,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}