{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 158,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "debug/policy_chosen_logits": 1.5589828491210938,
      "debug/policy_chosen_logps": -258.5330810546875,
      "debug/policy_rejected_logits": 1.9977812767028809,
      "debug/policy_rejected_logps": -304.0617980957031,
      "debug/reference_chosen_logps": -258.5330810546875,
      "debug/reference_rejected_logps": -304.0617980957031,
      "epoch": 0.006329113924050633,
      "grad_norm": 5.915865288930895,
      "learning_rate": 1e-06,
      "logits/chosen": 1.5589828491210938,
      "logits/rejected": 1.9977812767028809,
      "logps/chosen": -258.5330810546875,
      "logps/rejected": -304.0617980957031,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "debug/policy_chosen_logits": 0.8514629602432251,
      "debug/policy_chosen_logps": -222.75827026367188,
      "debug/policy_rejected_logits": 1.458482027053833,
      "debug/policy_rejected_logps": -292.2978210449219,
      "debug/reference_chosen_logps": -222.56484985351562,
      "debug/reference_rejected_logps": -288.334716796875,
      "epoch": 0.03164556962025317,
      "grad_norm": 6.379094662882782,
      "learning_rate": 1e-06,
      "logits/chosen": 0.8514629602432251,
      "logits/rejected": 1.458482027053833,
      "logps/chosen": -222.75827026367188,
      "logps/rejected": -292.2978210449219,
      "loss": 0.4816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0019342182204127312,
      "rewards/margins": 0.03769642859697342,
      "rewards/rejected": -0.039630644023418427,
      "step": 5
    },
    {
      "debug/policy_chosen_logits": 1.1418471336364746,
      "debug/policy_chosen_logps": -261.1085510253906,
      "debug/policy_rejected_logits": 1.316489338874817,
      "debug/policy_rejected_logps": -285.4795837402344,
      "debug/reference_chosen_logps": -260.5736999511719,
      "debug/reference_rejected_logps": -280.2572937011719,
      "epoch": 0.06329113924050633,
      "grad_norm": 12.885197123935471,
      "learning_rate": 1e-06,
      "logits/chosen": 1.1418471336364746,
      "logits/rejected": 1.316489338874817,
      "logps/chosen": -261.1085510253906,
      "logps/rejected": -285.4795837402344,
      "loss": 0.4629,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.005348391830921173,
      "rewards/margins": 0.04687455669045448,
      "rewards/rejected": -0.05222295597195625,
      "step": 10
    },
    {
      "debug/policy_chosen_logits": 1.4202030897140503,
      "debug/policy_chosen_logps": -305.30096435546875,
      "debug/policy_rejected_logits": 1.608795404434204,
      "debug/policy_rejected_logps": -339.3628845214844,
      "debug/reference_chosen_logps": -305.89739990234375,
      "debug/reference_rejected_logps": -336.0830078125,
      "epoch": 0.0949367088607595,
      "grad_norm": 6.031873391940916,
      "learning_rate": 1e-06,
      "logits/chosen": 1.4202030897140503,
      "logits/rejected": 1.608795404434204,
      "logps/chosen": -305.30096435546875,
      "logps/rejected": -339.3628845214844,
      "loss": 0.462,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.005964324809610844,
      "rewards/margins": 0.038763098418712616,
      "rewards/rejected": -0.032798778265714645,
      "step": 15
    },
    {
      "debug/policy_chosen_logits": 1.2072508335113525,
      "debug/policy_chosen_logps": -259.9560546875,
      "debug/policy_rejected_logits": 1.4596980810165405,
      "debug/policy_rejected_logps": -266.99896240234375,
      "debug/reference_chosen_logps": -262.2249450683594,
      "debug/reference_rejected_logps": -262.94488525390625,
      "epoch": 0.12658227848101267,
      "grad_norm": 5.929430664241562,
      "learning_rate": 1e-06,
      "logits/chosen": 1.2072508335113525,
      "logits/rejected": 1.4596980810165405,
      "logps/chosen": -259.9560546875,
      "logps/rejected": -266.99896240234375,
      "loss": 0.4568,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02268880605697632,
      "rewards/margins": 0.06322960555553436,
      "rewards/rejected": -0.04054080322384834,
      "step": 20
    },
    {
      "debug/policy_chosen_logits": 0.9236510992050171,
      "debug/policy_chosen_logps": -263.56951904296875,
      "debug/policy_rejected_logits": 1.2153400182724,
      "debug/policy_rejected_logps": -276.596923828125,
      "debug/reference_chosen_logps": -264.62982177734375,
      "debug/reference_rejected_logps": -272.1346130371094,
      "epoch": 0.15822784810126583,
      "grad_norm": 6.795022163630081,
      "learning_rate": 1e-06,
      "logits/chosen": 0.9236510992050171,
      "logits/rejected": 1.2153400182724,
      "logps/chosen": -263.56951904296875,
      "logps/rejected": -276.596923828125,
      "loss": 0.4609,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.010603101924061775,
      "rewards/margins": 0.05522637441754341,
      "rewards/rejected": -0.04462327063083649,
      "step": 25
    },
    {
      "debug/policy_chosen_logits": 0.8845943212509155,
      "debug/policy_chosen_logps": -232.0923309326172,
      "debug/policy_rejected_logits": 1.284155011177063,
      "debug/policy_rejected_logps": -287.80389404296875,
      "debug/reference_chosen_logps": -233.78652954101562,
      "debug/reference_rejected_logps": -284.5167236328125,
      "epoch": 0.189873417721519,
      "grad_norm": 6.4445556777608255,
      "learning_rate": 1e-06,
      "logits/chosen": 0.8845943212509155,
      "logits/rejected": 1.284155011177063,
      "logps/chosen": -232.0923309326172,
      "logps/rejected": -287.80389404296875,
      "loss": 0.4609,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.016941774636507034,
      "rewards/margins": 0.04981378838419914,
      "rewards/rejected": -0.03287201002240181,
      "step": 30
    },
    {
      "debug/policy_chosen_logits": 1.1807546615600586,
      "debug/policy_chosen_logps": -263.7032165527344,
      "debug/policy_rejected_logits": 1.3615357875823975,
      "debug/policy_rejected_logps": -295.0924377441406,
      "debug/reference_chosen_logps": -264.52520751953125,
      "debug/reference_rejected_logps": -289.96612548828125,
      "epoch": 0.22151898734177214,
      "grad_norm": 6.39988158389298,
      "learning_rate": 1e-06,
      "logits/chosen": 1.1807546615600586,
      "logits/rejected": 1.3615357875823975,
      "logps/chosen": -263.7032165527344,
      "logps/rejected": -295.0924377441406,
      "loss": 0.4495,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.008219520561397076,
      "rewards/margins": 0.05948234722018242,
      "rewards/rejected": -0.05126282572746277,
      "step": 35
    },
    {
      "debug/policy_chosen_logits": 0.918303370475769,
      "debug/policy_chosen_logps": -224.531982421875,
      "debug/policy_rejected_logits": 1.2155705690383911,
      "debug/policy_rejected_logps": -266.7242431640625,
      "debug/reference_chosen_logps": -227.6628875732422,
      "debug/reference_rejected_logps": -259.6141052246094,
      "epoch": 0.25316455696202533,
      "grad_norm": 8.66786179216246,
      "learning_rate": 1e-06,
      "logits/chosen": 0.918303370475769,
      "logits/rejected": 1.2155705690383911,
      "logps/chosen": -224.531982421875,
      "logps/rejected": -266.7242431640625,
      "loss": 0.4495,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.031309086829423904,
      "rewards/margins": 0.10241049528121948,
      "rewards/rejected": -0.07110141217708588,
      "step": 40
    },
    {
      "debug/policy_chosen_logits": 0.8259471654891968,
      "debug/policy_chosen_logps": -230.60250854492188,
      "debug/policy_rejected_logits": 1.2626183032989502,
      "debug/policy_rejected_logps": -303.4950866699219,
      "debug/reference_chosen_logps": -230.0920867919922,
      "debug/reference_rejected_logps": -302.10784912109375,
      "epoch": 0.2848101265822785,
      "grad_norm": 6.143825464676947,
      "learning_rate": 1e-06,
      "logits/chosen": 0.8259471654891968,
      "logits/rejected": 1.2626183032989502,
      "logps/chosen": -230.60250854492188,
      "logps/rejected": -303.4950866699219,
      "loss": 0.4802,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0051041776314377785,
      "rewards/margins": 0.008768384344875813,
      "rewards/rejected": -0.013872561976313591,
      "step": 45
    },
    {
      "debug/policy_chosen_logits": 0.9409104585647583,
      "debug/policy_chosen_logps": -241.2617950439453,
      "debug/policy_rejected_logits": 1.2857184410095215,
      "debug/policy_rejected_logps": -291.4665222167969,
      "debug/reference_chosen_logps": -244.69577026367188,
      "debug/reference_rejected_logps": -284.1947021484375,
      "epoch": 0.31645569620253167,
      "grad_norm": 8.46649937885156,
      "learning_rate": 1e-06,
      "logits/chosen": 0.9409104585647583,
      "logits/rejected": 1.2857184410095215,
      "logps/chosen": -241.2617950439453,
      "logps/rejected": -291.4665222167969,
      "loss": 0.4411,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.034339673817157745,
      "rewards/margins": 0.10705772787332535,
      "rewards/rejected": -0.0727180689573288,
      "step": 50
    },
    {
      "debug/policy_chosen_logits": 0.8741863369941711,
      "debug/policy_chosen_logps": -250.87057495117188,
      "debug/policy_rejected_logits": 1.258837103843689,
      "debug/policy_rejected_logps": -289.27069091796875,
      "debug/reference_chosen_logps": -255.7415771484375,
      "debug/reference_rejected_logps": -283.4430847167969,
      "epoch": 0.34810126582278483,
      "grad_norm": 9.716442001601763,
      "learning_rate": 1e-06,
      "logits/chosen": 0.8741863369941711,
      "logits/rejected": 1.258837103843689,
      "logps/chosen": -250.87057495117188,
      "logps/rejected": -289.27069091796875,
      "loss": 0.4436,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.048710085451602936,
      "rewards/margins": 0.10698604583740234,
      "rewards/rejected": -0.05827596038579941,
      "step": 55
    },
    {
      "debug/policy_chosen_logits": 0.6640017628669739,
      "debug/policy_chosen_logps": -269.62237548828125,
      "debug/policy_rejected_logits": 0.8445190191268921,
      "debug/policy_rejected_logps": -291.27325439453125,
      "debug/reference_chosen_logps": -269.4212951660156,
      "debug/reference_rejected_logps": -285.77349853515625,
      "epoch": 0.379746835443038,
      "grad_norm": 7.925495242886814,
      "learning_rate": 1e-06,
      "logits/chosen": 0.6640017628669739,
      "logits/rejected": 0.8445190191268921,
      "logps/chosen": -269.62237548828125,
      "logps/rejected": -291.27325439453125,
      "loss": 0.438,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.002010857220739126,
      "rewards/margins": 0.052986472845077515,
      "rewards/rejected": -0.05499732494354248,
      "step": 60
    },
    {
      "debug/policy_chosen_logits": 1.0082881450653076,
      "debug/policy_chosen_logps": -241.1085662841797,
      "debug/policy_rejected_logits": 1.5921090841293335,
      "debug/policy_rejected_logps": -303.08465576171875,
      "debug/reference_chosen_logps": -245.0981903076172,
      "debug/reference_rejected_logps": -300.36328125,
      "epoch": 0.41139240506329117,
      "grad_norm": 7.096776814684128,
      "learning_rate": 1e-06,
      "logits/chosen": 1.0082881450653076,
      "logits/rejected": 1.5921090841293335,
      "logps/chosen": -241.1085662841797,
      "logps/rejected": -303.08465576171875,
      "loss": 0.4602,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03989603370428085,
      "rewards/margins": 0.06710983067750931,
      "rewards/rejected": -0.027213791385293007,
      "step": 65
    },
    {
      "debug/policy_chosen_logits": 0.7952272295951843,
      "debug/policy_chosen_logps": -252.08798217773438,
      "debug/policy_rejected_logits": 1.0696840286254883,
      "debug/policy_rejected_logps": -287.27301025390625,
      "debug/reference_chosen_logps": -253.79379272460938,
      "debug/reference_rejected_logps": -279.5188903808594,
      "epoch": 0.4430379746835443,
      "grad_norm": 7.584678181203943,
      "learning_rate": 1e-06,
      "logits/chosen": 0.7952272295951843,
      "logits/rejected": 1.0696840286254883,
      "logps/chosen": -252.08798217773438,
      "logps/rejected": -287.27301025390625,
      "loss": 0.4335,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.017058206722140312,
      "rewards/margins": 0.09459935128688812,
      "rewards/rejected": -0.07754113525152206,
      "step": 70
    },
    {
      "debug/policy_chosen_logits": 0.9075101613998413,
      "debug/policy_chosen_logps": -218.43185424804688,
      "debug/policy_rejected_logits": 1.0321990251541138,
      "debug/policy_rejected_logps": -245.87973022460938,
      "debug/reference_chosen_logps": -221.93466186523438,
      "debug/reference_rejected_logps": -243.0590057373047,
      "epoch": 0.47468354430379744,
      "grad_norm": 6.725884442562555,
      "learning_rate": 1e-06,
      "logits/chosen": 0.9075101613998413,
      "logits/rejected": 1.0321990251541138,
      "logps/chosen": -218.43185424804688,
      "logps/rejected": -245.87973022460938,
      "loss": 0.4441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03502799943089485,
      "rewards/margins": 0.06323517113924026,
      "rewards/rejected": -0.028207167983055115,
      "step": 75
    },
    {
      "debug/policy_chosen_logits": 0.6510931253433228,
      "debug/policy_chosen_logps": -218.7671356201172,
      "debug/policy_rejected_logits": 0.8215225338935852,
      "debug/policy_rejected_logps": -276.33111572265625,
      "debug/reference_chosen_logps": -222.28018188476562,
      "debug/reference_rejected_logps": -267.1961364746094,
      "epoch": 0.5063291139240507,
      "grad_norm": 7.155350358859657,
      "learning_rate": 1e-06,
      "logits/chosen": 0.6510931253433228,
      "logits/rejected": 0.8215225338935852,
      "logps/chosen": -218.7671356201172,
      "logps/rejected": -276.33111572265625,
      "loss": 0.4348,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.03513062372803688,
      "rewards/margins": 0.1264806091785431,
      "rewards/rejected": -0.09134997427463531,
      "step": 80
    },
    {
      "debug/policy_chosen_logits": 0.9534305334091187,
      "debug/policy_chosen_logps": -250.000244140625,
      "debug/policy_rejected_logits": 1.0431879758834839,
      "debug/policy_rejected_logps": -275.9551086425781,
      "debug/reference_chosen_logps": -250.7502899169922,
      "debug/reference_rejected_logps": -268.43548583984375,
      "epoch": 0.5379746835443038,
      "grad_norm": 26.837408837144096,
      "learning_rate": 1e-06,
      "logits/chosen": 0.9534305334091187,
      "logits/rejected": 1.0431879758834839,
      "logps/chosen": -250.000244140625,
      "logps/rejected": -275.9551086425781,
      "loss": 0.4926,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.007500249892473221,
      "rewards/margins": 0.08269646763801575,
      "rewards/rejected": -0.07519622147083282,
      "step": 85
    },
    {
      "debug/policy_chosen_logits": 1.1253650188446045,
      "debug/policy_chosen_logps": -240.8356475830078,
      "debug/policy_rejected_logits": 1.2428481578826904,
      "debug/policy_rejected_logps": -265.67266845703125,
      "debug/reference_chosen_logps": -245.643798828125,
      "debug/reference_rejected_logps": -261.6888122558594,
      "epoch": 0.569620253164557,
      "grad_norm": 8.938690009286978,
      "learning_rate": 1e-06,
      "logits/chosen": 1.1253650188446045,
      "logits/rejected": 1.2428481578826904,
      "logps/chosen": -240.8356475830078,
      "logps/rejected": -265.67266845703125,
      "loss": 0.4314,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.04808169603347778,
      "rewards/margins": 0.08791980892419815,
      "rewards/rejected": -0.03983811289072037,
      "step": 90
    },
    {
      "debug/policy_chosen_logits": 0.9913564920425415,
      "debug/policy_chosen_logps": -247.68453979492188,
      "debug/policy_rejected_logits": 1.167474389076233,
      "debug/policy_rejected_logps": -284.51300048828125,
      "debug/reference_chosen_logps": -250.7725067138672,
      "debug/reference_rejected_logps": -276.8506774902344,
      "epoch": 0.6012658227848101,
      "grad_norm": 7.214786092625251,
      "learning_rate": 1e-06,
      "logits/chosen": 0.9913564920425415,
      "logits/rejected": 1.167474389076233,
      "logps/chosen": -247.68453979492188,
      "logps/rejected": -284.51300048828125,
      "loss": 0.4481,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.030879342928528786,
      "rewards/margins": 0.10750222206115723,
      "rewards/rejected": -0.07662288844585419,
      "step": 95
    },
    {
      "debug/policy_chosen_logits": 1.548004388809204,
      "debug/policy_chosen_logps": -286.9696350097656,
      "debug/policy_rejected_logits": 1.2569023370742798,
      "debug/policy_rejected_logps": -255.9474639892578,
      "debug/reference_chosen_logps": -288.26263427734375,
      "debug/reference_rejected_logps": -252.56982421875,
      "epoch": 0.6329113924050633,
      "grad_norm": 7.098617456221662,
      "learning_rate": 1e-06,
      "logits/chosen": 1.548004388809204,
      "logits/rejected": 1.2569023370742798,
      "logps/chosen": -286.9696350097656,
      "logps/rejected": -255.9474639892578,
      "loss": 0.4429,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012930279597640038,
      "rewards/margins": 0.04670674726366997,
      "rewards/rejected": -0.03377646952867508,
      "step": 100
    },
    {
      "epoch": 0.6329113924050633,
      "eval_debug/policy_chosen_logits": 1.2252188920974731,
      "eval_debug/policy_chosen_logps": -250.68939208984375,
      "eval_debug/policy_rejected_logits": 1.4343616962432861,
      "eval_debug/policy_rejected_logps": -287.45086669921875,
      "eval_debug/reference_chosen_logps": -255.34970092773438,
      "eval_debug/reference_rejected_logps": -283.57049560546875,
      "eval_logits/chosen": 1.2252188920974731,
      "eval_logits/rejected": 1.4343616962432861,
      "eval_logps/chosen": -250.68939208984375,
      "eval_logps/rejected": -287.45086669921875,
      "eval_loss": 0.43653252720832825,
      "eval_rewards/accuracies": 0.5769230723381042,
      "eval_rewards/chosen": 0.04660310223698616,
      "eval_rewards/margins": 0.08540700376033783,
      "eval_rewards/rejected": -0.03880389407277107,
      "eval_runtime": 19.8549,
      "eval_samples_per_second": 20.146,
      "eval_steps_per_second": 0.655,
      "step": 100
    },
    {
      "debug/policy_chosen_logits": 1.011919617652893,
      "debug/policy_chosen_logps": -279.73260498046875,
      "debug/policy_rejected_logits": 1.211625337600708,
      "debug/policy_rejected_logps": -298.412109375,
      "debug/reference_chosen_logps": -281.5310974121094,
      "debug/reference_rejected_logps": -292.20550537109375,
      "epoch": 0.6645569620253164,
      "grad_norm": 6.340425768293679,
      "learning_rate": 1e-06,
      "logits/chosen": 1.011919617652893,
      "logits/rejected": 1.211625337600708,
      "logps/chosen": -279.73260498046875,
      "logps/rejected": -298.412109375,
      "loss": 0.4362,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.017984820529818535,
      "rewards/margins": 0.08005066215991974,
      "rewards/rejected": -0.06206584721803665,
      "step": 105
    },
    {
      "debug/policy_chosen_logits": 1.0565037727355957,
      "debug/policy_chosen_logps": -251.0978546142578,
      "debug/policy_rejected_logits": 1.3947855234146118,
      "debug/policy_rejected_logps": -316.4710998535156,
      "debug/reference_chosen_logps": -253.4007110595703,
      "debug/reference_rejected_logps": -309.9458923339844,
      "epoch": 0.6962025316455697,
      "grad_norm": 20.34165260676491,
      "learning_rate": 1e-06,
      "logits/chosen": 1.0565037727355957,
      "logits/rejected": 1.3947855234146118,
      "logps/chosen": -251.0978546142578,
      "logps/rejected": -316.4710998535156,
      "loss": 0.4383,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.023028511554002762,
      "rewards/margins": 0.08828048408031464,
      "rewards/rejected": -0.06525196880102158,
      "step": 110
    },
    {
      "debug/policy_chosen_logits": 0.8845629692077637,
      "debug/policy_chosen_logps": -241.9716339111328,
      "debug/policy_rejected_logits": 1.229775071144104,
      "debug/policy_rejected_logps": -321.60186767578125,
      "debug/reference_chosen_logps": -246.28433227539062,
      "debug/reference_rejected_logps": -314.5198974609375,
      "epoch": 0.7278481012658228,
      "grad_norm": 7.789166803514712,
      "learning_rate": 1e-06,
      "logits/chosen": 0.8845629692077637,
      "logits/rejected": 1.229775071144104,
      "logps/chosen": -241.9716339111328,
      "logps/rejected": -321.60186767578125,
      "loss": 0.4426,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.043126728385686874,
      "rewards/margins": 0.11394629627466202,
      "rewards/rejected": -0.07081956416368484,
      "step": 115
    },
    {
      "debug/policy_chosen_logits": 0.6471331119537354,
      "debug/policy_chosen_logps": -232.4429168701172,
      "debug/policy_rejected_logits": 0.9131924510002136,
      "debug/policy_rejected_logps": -279.41290283203125,
      "debug/reference_chosen_logps": -237.39102172851562,
      "debug/reference_rejected_logps": -273.61090087890625,
      "epoch": 0.759493670886076,
      "grad_norm": 7.468046301754059,
      "learning_rate": 1e-06,
      "logits/chosen": 0.6471331119537354,
      "logits/rejected": 0.9131924510002136,
      "logps/chosen": -232.4429168701172,
      "logps/rejected": -279.41290283203125,
      "loss": 0.4131,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04948071017861366,
      "rewards/margins": 0.1075005754828453,
      "rewards/rejected": -0.05801987648010254,
      "step": 120
    },
    {
      "debug/policy_chosen_logits": 0.9338349103927612,
      "debug/policy_chosen_logps": -260.35235595703125,
      "debug/policy_rejected_logits": 1.0534359216690063,
      "debug/policy_rejected_logps": -297.56683349609375,
      "debug/reference_chosen_logps": -264.9391174316406,
      "debug/reference_rejected_logps": -289.8217468261719,
      "epoch": 0.7911392405063291,
      "grad_norm": 8.935461685140815,
      "learning_rate": 1e-06,
      "logits/chosen": 0.9338349103927612,
      "logits/rejected": 1.0534359216690063,
      "logps/chosen": -260.35235595703125,
      "logps/rejected": -297.56683349609375,
      "loss": 0.4303,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.045867711305618286,
      "rewards/margins": 0.12331867218017578,
      "rewards/rejected": -0.0774509608745575,
      "step": 125
    },
    {
      "debug/policy_chosen_logits": 0.8780291676521301,
      "debug/policy_chosen_logps": -284.29205322265625,
      "debug/policy_rejected_logits": 0.8824840784072876,
      "debug/policy_rejected_logps": -287.76690673828125,
      "debug/reference_chosen_logps": -286.41943359375,
      "debug/reference_rejected_logps": -283.56903076171875,
      "epoch": 0.8227848101265823,
      "grad_norm": 6.948216331668783,
      "learning_rate": 1e-06,
      "logits/chosen": 0.8780291676521301,
      "logits/rejected": 0.8824840784072876,
      "logps/chosen": -284.29205322265625,
      "logps/rejected": -287.76690673828125,
      "loss": 0.4375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02127380482852459,
      "rewards/margins": 0.06325232237577438,
      "rewards/rejected": -0.041978511959314346,
      "step": 130
    },
    {
      "debug/policy_chosen_logits": 1.165907859802246,
      "debug/policy_chosen_logps": -255.9198455810547,
      "debug/policy_rejected_logits": 1.4020473957061768,
      "debug/policy_rejected_logps": -301.6413879394531,
      "debug/reference_chosen_logps": -260.84521484375,
      "debug/reference_rejected_logps": -295.99700927734375,
      "epoch": 0.8544303797468354,
      "grad_norm": 6.0797186914906485,
      "learning_rate": 1e-06,
      "logits/chosen": 1.165907859802246,
      "logits/rejected": 1.4020473957061768,
      "logps/chosen": -255.9198455810547,
      "logps/rejected": -301.6413879394531,
      "loss": 0.4418,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.049253594130277634,
      "rewards/margins": 0.10569741576910019,
      "rewards/rejected": -0.056443821638822556,
      "step": 135
    },
    {
      "debug/policy_chosen_logits": 0.9684173464775085,
      "debug/policy_chosen_logps": -240.7368927001953,
      "debug/policy_rejected_logits": 1.522164225578308,
      "debug/policy_rejected_logps": -300.8490295410156,
      "debug/reference_chosen_logps": -244.41757202148438,
      "debug/reference_rejected_logps": -289.0794372558594,
      "epoch": 0.8860759493670886,
      "grad_norm": 6.850074566718433,
      "learning_rate": 1e-06,
      "logits/chosen": 0.9684173464775085,
      "logits/rejected": 1.522164225578308,
      "logps/chosen": -240.7368927001953,
      "logps/rejected": -300.8490295410156,
      "loss": 0.43,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03680698946118355,
      "rewards/margins": 0.1545029729604721,
      "rewards/rejected": -0.11769597232341766,
      "step": 140
    },
    {
      "debug/policy_chosen_logits": 1.095474123954773,
      "debug/policy_chosen_logps": -281.7500305175781,
      "debug/policy_rejected_logits": 1.0368950366973877,
      "debug/policy_rejected_logps": -281.8016052246094,
      "debug/reference_chosen_logps": -285.4373474121094,
      "debug/reference_rejected_logps": -278.67181396484375,
      "epoch": 0.9177215189873418,
      "grad_norm": 6.330596887372699,
      "learning_rate": 1e-06,
      "logits/chosen": 1.095474123954773,
      "logits/rejected": 1.0368950366973877,
      "logps/chosen": -281.7500305175781,
      "logps/rejected": -281.8016052246094,
      "loss": 0.4243,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.036873430013656616,
      "rewards/margins": 0.06817178428173065,
      "rewards/rejected": -0.03129836544394493,
      "step": 145
    },
    {
      "debug/policy_chosen_logits": 0.9509929418563843,
      "debug/policy_chosen_logps": -247.018310546875,
      "debug/policy_rejected_logits": 1.1111629009246826,
      "debug/policy_rejected_logps": -272.07684326171875,
      "debug/reference_chosen_logps": -250.40658569335938,
      "debug/reference_rejected_logps": -265.6427001953125,
      "epoch": 0.9493670886075949,
      "grad_norm": 8.073046871358697,
      "learning_rate": 1e-06,
      "logits/chosen": 0.9509929418563843,
      "logits/rejected": 1.1111629009246826,
      "logps/chosen": -247.018310546875,
      "logps/rejected": -272.07684326171875,
      "loss": 0.4234,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03388286381959915,
      "rewards/margins": 0.09822405129671097,
      "rewards/rejected": -0.06434118002653122,
      "step": 150
    },
    {
      "debug/policy_chosen_logits": 0.6622827053070068,
      "debug/policy_chosen_logps": -237.2403106689453,
      "debug/policy_rejected_logits": 0.8520939946174622,
      "debug/policy_rejected_logps": -286.5059509277344,
      "debug/reference_chosen_logps": -241.94467163085938,
      "debug/reference_rejected_logps": -278.73272705078125,
      "epoch": 0.9810126582278481,
      "grad_norm": 7.904037537559287,
      "learning_rate": 1e-06,
      "logits/chosen": 0.6622827053070068,
      "logits/rejected": 0.8520939946174622,
      "logps/chosen": -237.2403106689453,
      "logps/rejected": -286.5059509277344,
      "loss": 0.423,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.04704369604587555,
      "rewards/margins": 0.12477605044841766,
      "rewards/rejected": -0.07773236930370331,
      "step": 155
    },
    {
      "epoch": 1.0,
      "step": 158,
      "total_flos": 0.0,
      "train_loss": 0.44511839181562013,
      "train_runtime": 1281.3009,
      "train_samples_per_second": 7.867,
      "train_steps_per_second": 0.123
    }
  ],
  "logging_steps": 5,
  "max_steps": 158,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}