{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 48,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "debug/policy_chosen_logits": 32.32257843017578,
      "debug/policy_chosen_logps": -428.3227844238281,
      "debug/policy_rejected_logits": 30.82423210144043,
      "debug/policy_rejected_logps": -426.3280029296875,
      "debug/reference_chosen_logps": -428.3227844238281,
      "debug/reference_rejected_logps": -426.3280029296875,
      "epoch": 0.020833333333333332,
      "grad_norm": 6.447932275019588,
      "learning_rate": 1e-06,
      "logits/chosen": 32.32257843017578,
      "logits/rejected": 30.82423210144043,
      "logps/chosen": -428.3227844238281,
      "logps/rejected": -426.3280029296875,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "debug/policy_chosen_logits": 31.886442184448242,
      "debug/policy_chosen_logps": -418.96539306640625,
      "debug/policy_rejected_logits": 29.381662368774414,
      "debug/policy_rejected_logps": -429.94775390625,
      "debug/reference_chosen_logps": -418.5838623046875,
      "debug/reference_rejected_logps": -429.845703125,
      "epoch": 0.041666666666666664,
      "grad_norm": 5.235639772033975,
      "learning_rate": 1e-06,
      "logits/chosen": 31.886442184448242,
      "logits/rejected": 29.381662368774414,
      "logps/chosen": -418.96539306640625,
      "logps/rejected": -429.94775390625,
      "loss": 0.4988,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.00381515477783978,
      "rewards/margins": -0.0027948375791311264,
      "rewards/rejected": -0.001020316849462688,
      "step": 2
    },
    {
      "debug/policy_chosen_logits": 31.19687843322754,
      "debug/policy_chosen_logps": -417.73297119140625,
      "debug/policy_rejected_logits": 29.822526931762695,
      "debug/policy_rejected_logps": -424.8887939453125,
      "debug/reference_chosen_logps": -416.591796875,
      "debug/reference_rejected_logps": -424.60333251953125,
      "epoch": 0.0625,
      "grad_norm": 5.035431279495221,
      "learning_rate": 1e-06,
      "logits/chosen": 31.19687843322754,
      "logits/rejected": 29.822526931762695,
      "logps/chosen": -417.73297119140625,
      "logps/rejected": -424.8887939453125,
      "loss": 0.5013,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.011411934159696102,
      "rewards/margins": -0.008557281456887722,
      "rewards/rejected": -0.002854652237147093,
      "step": 3
    },
    {
      "debug/policy_chosen_logits": 31.30547523498535,
      "debug/policy_chosen_logps": -409.65521240234375,
      "debug/policy_rejected_logits": 31.427326202392578,
      "debug/policy_rejected_logps": -419.1351013183594,
      "debug/reference_chosen_logps": -410.0574951171875,
      "debug/reference_rejected_logps": -419.167724609375,
      "epoch": 0.08333333333333333,
      "grad_norm": 4.947716251775631,
      "learning_rate": 1e-06,
      "logits/chosen": 31.30547523498535,
      "logits/rejected": 31.427326202392578,
      "logps/chosen": -409.65521240234375,
      "logps/rejected": -419.1351013183594,
      "loss": 0.4997,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0040228646248579025,
      "rewards/margins": 0.0036964411847293377,
      "rewards/rejected": 0.00032642344012856483,
      "step": 4
    },
    {
      "debug/policy_chosen_logits": 26.84766387939453,
      "debug/policy_chosen_logps": -419.80255126953125,
      "debug/policy_rejected_logits": 25.351024627685547,
      "debug/policy_rejected_logps": -403.54278564453125,
      "debug/reference_chosen_logps": -419.79803466796875,
      "debug/reference_rejected_logps": -403.3204040527344,
      "epoch": 0.10416666666666667,
      "grad_norm": 5.437518779758296,
      "learning_rate": 1e-06,
      "logits/chosen": 26.84766387939453,
      "logits/rejected": 25.351024627685547,
      "logps/chosen": -419.80255126953125,
      "logps/rejected": -403.54278564453125,
      "loss": 0.5011,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.535634070634842e-05,
      "rewards/margins": 0.0021785348653793335,
      "rewards/rejected": -0.0022238921374082565,
      "step": 5
    },
    {
      "debug/policy_chosen_logits": 28.140277862548828,
      "debug/policy_chosen_logps": -435.39581298828125,
      "debug/policy_rejected_logits": 29.35475730895996,
      "debug/policy_rejected_logps": -438.25970458984375,
      "debug/reference_chosen_logps": -435.1541748046875,
      "debug/reference_rejected_logps": -436.6419677734375,
      "epoch": 0.125,
      "grad_norm": 5.3788783078874625,
      "learning_rate": 1e-06,
      "logits/chosen": 28.140277862548828,
      "logits/rejected": 29.35475730895996,
      "logps/chosen": -435.39581298828125,
      "logps/rejected": -438.25970458984375,
      "loss": 0.4935,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.002416381612420082,
      "rewards/margins": 0.013761366717517376,
      "rewards/rejected": -0.016177749261260033,
      "step": 6
    },
    {
      "debug/policy_chosen_logits": 32.219444274902344,
      "debug/policy_chosen_logps": -422.5959167480469,
      "debug/policy_rejected_logits": 33.60882568359375,
      "debug/policy_rejected_logps": -429.6781311035156,
      "debug/reference_chosen_logps": -422.28350830078125,
      "debug/reference_rejected_logps": -429.510009765625,
      "epoch": 0.14583333333333334,
      "grad_norm": 5.089230644656757,
      "learning_rate": 1e-06,
      "logits/chosen": 32.219444274902344,
      "logits/rejected": 33.60882568359375,
      "logps/chosen": -422.5959167480469,
      "logps/rejected": -429.6781311035156,
      "loss": 0.499,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0031237031798809767,
      "rewards/margins": -0.0014425661647692323,
      "rewards/rejected": -0.0016811370151117444,
      "step": 7
    },
    {
      "debug/policy_chosen_logits": 34.4580192565918,
      "debug/policy_chosen_logps": -425.48052978515625,
      "debug/policy_rejected_logits": 34.77037048339844,
      "debug/policy_rejected_logps": -458.2679138183594,
      "debug/reference_chosen_logps": -425.6324462890625,
      "debug/reference_rejected_logps": -455.59991455078125,
      "epoch": 0.16666666666666666,
      "grad_norm": 6.722332782296524,
      "learning_rate": 1e-06,
      "logits/chosen": 34.4580192565918,
      "logits/rejected": 34.77037048339844,
      "logps/chosen": -425.48052978515625,
      "logps/rejected": -458.2679138183594,
      "loss": 0.4938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0015194700099527836,
      "rewards/margins": 0.02819930762052536,
      "rewards/rejected": -0.026679838076233864,
      "step": 8
    },
    {
      "debug/policy_chosen_logits": 29.579063415527344,
      "debug/policy_chosen_logps": -397.6627502441406,
      "debug/policy_rejected_logits": 29.510604858398438,
      "debug/policy_rejected_logps": -423.0687255859375,
      "debug/reference_chosen_logps": -398.0777282714844,
      "debug/reference_rejected_logps": -422.35186767578125,
      "epoch": 0.1875,
      "grad_norm": 5.150446773896013,
      "learning_rate": 1e-06,
      "logits/chosen": 29.579063415527344,
      "logits/rejected": 29.510604858398438,
      "logps/chosen": -397.6627502441406,
      "logps/rejected": -423.0687255859375,
      "loss": 0.4984,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004149818792939186,
      "rewards/margins": 0.011318511329591274,
      "rewards/rejected": -0.007168693467974663,
      "step": 9
    },
    {
      "debug/policy_chosen_logits": 30.510459899902344,
      "debug/policy_chosen_logps": -406.46258544921875,
      "debug/policy_rejected_logits": 32.81418228149414,
      "debug/policy_rejected_logps": -420.8358154296875,
      "debug/reference_chosen_logps": -405.9990539550781,
      "debug/reference_rejected_logps": -419.8515930175781,
      "epoch": 0.20833333333333334,
      "grad_norm": 5.292871534847833,
      "learning_rate": 1e-06,
      "logits/chosen": 30.510459899902344,
      "logits/rejected": 32.81418228149414,
      "logps/chosen": -406.46258544921875,
      "logps/rejected": -420.8358154296875,
      "loss": 0.4965,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004634971730411053,
      "rewards/margins": 0.005207213573157787,
      "rewards/rejected": -0.009842186234891415,
      "step": 10
    },
    {
      "debug/policy_chosen_logits": 29.758987426757812,
      "debug/policy_chosen_logps": -421.1949462890625,
      "debug/policy_rejected_logits": 30.991361618041992,
      "debug/policy_rejected_logps": -440.055908203125,
      "debug/reference_chosen_logps": -420.9813537597656,
      "debug/reference_rejected_logps": -440.38330078125,
      "epoch": 0.22916666666666666,
      "grad_norm": 5.257332824569119,
      "learning_rate": 1e-06,
      "logits/chosen": 29.758987426757812,
      "logits/rejected": 30.991361618041992,
      "logps/chosen": -421.1949462890625,
      "logps/rejected": -440.055908203125,
      "loss": 0.4898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002135887276381254,
      "rewards/margins": -0.005409927107393742,
      "rewards/rejected": 0.003274040063843131,
      "step": 11
    },
    {
      "debug/policy_chosen_logits": 32.33553695678711,
      "debug/policy_chosen_logps": -442.6135559082031,
      "debug/policy_rejected_logits": 32.11320114135742,
      "debug/policy_rejected_logps": -444.33697509765625,
      "debug/reference_chosen_logps": -441.39215087890625,
      "debug/reference_rejected_logps": -443.31390380859375,
      "epoch": 0.25,
      "grad_norm": 6.359103183966066,
      "learning_rate": 1e-06,
      "logits/chosen": 32.33553695678711,
      "logits/rejected": 32.11320114135742,
      "logps/chosen": -442.6135559082031,
      "logps/rejected": -444.33697509765625,
      "loss": 0.4997,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.01221416424959898,
      "rewards/margins": -0.001983833033591509,
      "rewards/rejected": -0.010230331681668758,
      "step": 12
    },
    {
      "debug/policy_chosen_logits": 29.988950729370117,
      "debug/policy_chosen_logps": -425.54913330078125,
      "debug/policy_rejected_logits": 30.10309410095215,
      "debug/policy_rejected_logps": -426.21905517578125,
      "debug/reference_chosen_logps": -426.2005920410156,
      "debug/reference_rejected_logps": -424.59246826171875,
      "epoch": 0.2708333333333333,
      "grad_norm": 5.288806638849553,
      "learning_rate": 1e-06,
      "logits/chosen": 29.988950729370117,
      "logits/rejected": 30.10309410095215,
      "logps/chosen": -425.54913330078125,
      "logps/rejected": -426.21905517578125,
      "loss": 0.4925,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006514891982078552,
      "rewards/margins": 0.022780990228056908,
      "rewards/rejected": -0.016266098245978355,
      "step": 13
    },
    {
      "debug/policy_chosen_logits": 31.317047119140625,
      "debug/policy_chosen_logps": -419.84283447265625,
      "debug/policy_rejected_logits": 32.35562515258789,
      "debug/policy_rejected_logps": -399.378173828125,
      "debug/reference_chosen_logps": -421.95867919921875,
      "debug/reference_rejected_logps": -399.3148193359375,
      "epoch": 0.2916666666666667,
      "grad_norm": 5.154675497648274,
      "learning_rate": 1e-06,
      "logits/chosen": 31.317047119140625,
      "logits/rejected": 32.35562515258789,
      "logps/chosen": -419.84283447265625,
      "logps/rejected": -399.378173828125,
      "loss": 0.4906,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.021158332005143166,
      "rewards/margins": 0.021792029961943626,
      "rewards/rejected": -0.0006336974911391735,
      "step": 14
    },
    {
      "debug/policy_chosen_logits": 29.34722328186035,
      "debug/policy_chosen_logps": -394.68389892578125,
      "debug/policy_rejected_logits": 27.714096069335938,
      "debug/policy_rejected_logps": -414.51715087890625,
      "debug/reference_chosen_logps": -395.9772033691406,
      "debug/reference_rejected_logps": -414.28857421875,
      "epoch": 0.3125,
      "grad_norm": 5.191304228736527,
      "learning_rate": 1e-06,
      "logits/chosen": 29.34722328186035,
      "logits/rejected": 27.714096069335938,
      "logps/chosen": -394.68389892578125,
      "logps/rejected": -414.51715087890625,
      "loss": 0.4844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012933043763041496,
      "rewards/margins": 0.015218695625662804,
      "rewards/rejected": -0.0022856518626213074,
      "step": 15
    },
    {
      "debug/policy_chosen_logits": 29.499605178833008,
      "debug/policy_chosen_logps": -416.2149658203125,
      "debug/policy_rejected_logits": 31.734092712402344,
      "debug/policy_rejected_logps": -420.2775573730469,
      "debug/reference_chosen_logps": -417.6040954589844,
      "debug/reference_rejected_logps": -421.9405822753906,
      "epoch": 0.3333333333333333,
      "grad_norm": 5.336647699178971,
      "learning_rate": 1e-06,
      "logits/chosen": 29.499605178833008,
      "logits/rejected": 31.734092712402344,
      "logps/chosen": -416.2149658203125,
      "logps/rejected": -420.2775573730469,
      "loss": 0.4909,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.013891411013901234,
      "rewards/margins": -0.002739105373620987,
      "rewards/rejected": 0.016630517318844795,
      "step": 16
    },
    {
      "debug/policy_chosen_logits": 30.53592872619629,
      "debug/policy_chosen_logps": -424.2174987792969,
      "debug/policy_rejected_logits": 28.613372802734375,
      "debug/policy_rejected_logps": -424.45184326171875,
      "debug/reference_chosen_logps": -425.0408020019531,
      "debug/reference_rejected_logps": -424.396728515625,
      "epoch": 0.3541666666666667,
      "grad_norm": 5.069961718294066,
      "learning_rate": 1e-06,
      "logits/chosen": 30.53592872619629,
      "logits/rejected": 28.613372802734375,
      "logps/chosen": -424.2174987792969,
      "logps/rejected": -424.45184326171875,
      "loss": 0.4845,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.008232764899730682,
      "rewards/margins": 0.00878402590751648,
      "rewards/rejected": -0.000551263801753521,
      "step": 17
    },
    {
      "debug/policy_chosen_logits": 32.6837272644043,
      "debug/policy_chosen_logps": -439.52508544921875,
      "debug/policy_rejected_logits": 30.927047729492188,
      "debug/policy_rejected_logps": -429.91094970703125,
      "debug/reference_chosen_logps": -441.05828857421875,
      "debug/reference_rejected_logps": -431.7398376464844,
      "epoch": 0.375,
      "grad_norm": 5.035721183912941,
      "learning_rate": 1e-06,
      "logits/chosen": 32.6837272644043,
      "logits/rejected": 30.927047729492188,
      "logps/chosen": -439.52508544921875,
      "logps/rejected": -429.91094970703125,
      "loss": 0.5027,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.015331955626606941,
      "rewards/margins": -0.0029569631442427635,
      "rewards/rejected": 0.01828891783952713,
      "step": 18
    },
    {
      "debug/policy_chosen_logits": 30.297531127929688,
      "debug/policy_chosen_logps": -402.0390319824219,
      "debug/policy_rejected_logits": 33.57741165161133,
      "debug/policy_rejected_logps": -410.6739501953125,
      "debug/reference_chosen_logps": -401.3625793457031,
      "debug/reference_rejected_logps": -410.6376037597656,
      "epoch": 0.3958333333333333,
      "grad_norm": 5.398638220710165,
      "learning_rate": 1e-06,
      "logits/chosen": 30.297531127929688,
      "logits/rejected": 33.57741165161133,
      "logps/chosen": -402.0390319824219,
      "logps/rejected": -410.6739501953125,
      "loss": 0.4776,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.00676448829472065,
      "rewards/margins": -0.006401100195944309,
      "rewards/rejected": -0.0003633880987763405,
      "step": 19
    },
    {
      "debug/policy_chosen_logits": 31.950498580932617,
      "debug/policy_chosen_logps": -414.8414306640625,
      "debug/policy_rejected_logits": 33.85417938232422,
      "debug/policy_rejected_logps": -425.31268310546875,
      "debug/reference_chosen_logps": -415.87957763671875,
      "debug/reference_rejected_logps": -425.8223876953125,
      "epoch": 0.4166666666666667,
      "grad_norm": 5.034205804652484,
      "learning_rate": 1e-06,
      "logits/chosen": 31.950498580932617,
      "logits/rejected": 33.85417938232422,
      "logps/chosen": -414.8414306640625,
      "logps/rejected": -425.31268310546875,
      "loss": 0.4901,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010381774976849556,
      "rewards/margins": 0.005284767597913742,
      "rewards/rejected": 0.0050970083102583885,
      "step": 20
    },
    {
      "debug/policy_chosen_logits": 31.606367111206055,
      "debug/policy_chosen_logps": -415.55767822265625,
      "debug/policy_rejected_logits": 32.42195510864258,
      "debug/policy_rejected_logps": -409.7369384765625,
      "debug/reference_chosen_logps": -417.48565673828125,
      "debug/reference_rejected_logps": -411.9869689941406,
      "epoch": 0.4375,
      "grad_norm": 4.815275729217639,
      "learning_rate": 1e-06,
      "logits/chosen": 31.606367111206055,
      "logits/rejected": 32.42195510864258,
      "logps/chosen": -415.55767822265625,
      "logps/rejected": -409.7369384765625,
      "loss": 0.4884,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01928028091788292,
      "rewards/margins": -0.0032197567634284496,
      "rewards/rejected": 0.022500038146972656,
      "step": 21
    },
    {
      "debug/policy_chosen_logits": 30.70886993408203,
      "debug/policy_chosen_logps": -433.744873046875,
      "debug/policy_rejected_logits": 26.203298568725586,
      "debug/policy_rejected_logps": -415.767822265625,
      "debug/reference_chosen_logps": -431.77203369140625,
      "debug/reference_rejected_logps": -414.41796875,
      "epoch": 0.4583333333333333,
      "grad_norm": 5.219226631225296,
      "learning_rate": 1e-06,
      "logits/chosen": 30.70886993408203,
      "logits/rejected": 26.203298568725586,
      "logps/chosen": -433.744873046875,
      "logps/rejected": -415.767822265625,
      "loss": 0.4748,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.019728660583496094,
      "rewards/margins": -0.006230468861758709,
      "rewards/rejected": -0.01349819079041481,
      "step": 22
    },
    {
      "debug/policy_chosen_logits": 30.573122024536133,
      "debug/policy_chosen_logps": -420.0906677246094,
      "debug/policy_rejected_logits": 31.557247161865234,
      "debug/policy_rejected_logps": -405.26837158203125,
      "debug/reference_chosen_logps": -422.8019714355469,
      "debug/reference_rejected_logps": -406.9261474609375,
      "epoch": 0.4791666666666667,
      "grad_norm": 4.90573818429262,
      "learning_rate": 1e-06,
      "logits/chosen": 30.573122024536133,
      "logits/rejected": 31.557247161865234,
      "logps/chosen": -420.0906677246094,
      "logps/rejected": -405.26837158203125,
      "loss": 0.4845,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.027113037183880806,
      "rewards/margins": 0.010535049252212048,
      "rewards/rejected": 0.016577987000346184,
      "step": 23
    },
    {
      "debug/policy_chosen_logits": 28.54072380065918,
      "debug/policy_chosen_logps": -429.51544189453125,
      "debug/policy_rejected_logits": 28.603498458862305,
      "debug/policy_rejected_logps": -448.6463623046875,
      "debug/reference_chosen_logps": -429.3318176269531,
      "debug/reference_rejected_logps": -445.1684875488281,
      "epoch": 0.5,
      "grad_norm": 4.919240212142553,
      "learning_rate": 1e-06,
      "logits/chosen": 28.54072380065918,
      "logits/rejected": 28.603498458862305,
      "logps/chosen": -429.51544189453125,
      "logps/rejected": -448.6463623046875,
      "loss": 0.4835,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0018361280672252178,
      "rewards/margins": 0.03294284641742706,
      "rewards/rejected": -0.03477897495031357,
      "step": 24
    },
    {
      "debug/policy_chosen_logits": 32.81056594848633,
      "debug/policy_chosen_logps": -443.39068603515625,
      "debug/policy_rejected_logits": 30.637916564941406,
      "debug/policy_rejected_logps": -425.4708251953125,
      "debug/reference_chosen_logps": -439.9627685546875,
      "debug/reference_rejected_logps": -420.35076904296875,
      "epoch": 0.5208333333333334,
      "grad_norm": 5.047798852836749,
      "learning_rate": 1e-06,
      "logits/chosen": 32.81056594848633,
      "logits/rejected": 30.637916564941406,
      "logps/chosen": -443.39068603515625,
      "logps/rejected": -425.4708251953125,
      "loss": 0.4832,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03427928686141968,
      "rewards/margins": 0.016921542584896088,
      "rewards/rejected": -0.051200829446315765,
      "step": 25
    },
    {
      "debug/policy_chosen_logits": 29.654489517211914,
      "debug/policy_chosen_logps": -403.1080322265625,
      "debug/policy_rejected_logits": 29.024642944335938,
      "debug/policy_rejected_logps": -421.46270751953125,
      "debug/reference_chosen_logps": -404.3522644042969,
      "debug/reference_rejected_logps": -419.66693115234375,
      "epoch": 0.5416666666666666,
      "grad_norm": 4.941664000787305,
      "learning_rate": 1e-06,
      "logits/chosen": 29.654489517211914,
      "logits/rejected": 29.024642944335938,
      "logps/chosen": -403.1080322265625,
      "logps/rejected": -421.46270751953125,
      "loss": 0.4751,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012442246079444885,
      "rewards/margins": 0.030400237068533897,
      "rewards/rejected": -0.01795799285173416,
      "step": 26
    },
    {
      "debug/policy_chosen_logits": 28.045677185058594,
      "debug/policy_chosen_logps": -403.36614990234375,
      "debug/policy_rejected_logits": 30.181791305541992,
      "debug/policy_rejected_logps": -423.99591064453125,
      "debug/reference_chosen_logps": -405.4366149902344,
      "debug/reference_rejected_logps": -424.10650634765625,
      "epoch": 0.5625,
      "grad_norm": 5.109404536265639,
      "learning_rate": 1e-06,
      "logits/chosen": 28.045677185058594,
      "logits/rejected": 30.181791305541992,
      "logps/chosen": -403.36614990234375,
      "logps/rejected": -423.99591064453125,
      "loss": 0.4726,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.020704690366983414,
      "rewards/margins": 0.019598999992012978,
      "rewards/rejected": 0.0011056894436478615,
      "step": 27
    },
    {
      "debug/policy_chosen_logits": 30.849773406982422,
      "debug/policy_chosen_logps": -421.11431884765625,
      "debug/policy_rejected_logits": 30.653217315673828,
      "debug/policy_rejected_logps": -440.32244873046875,
      "debug/reference_chosen_logps": -418.60089111328125,
      "debug/reference_rejected_logps": -439.1358642578125,
      "epoch": 0.5833333333333334,
      "grad_norm": 5.1018343354077205,
      "learning_rate": 1e-06,
      "logits/chosen": 30.849773406982422,
      "logits/rejected": 30.653217315673828,
      "logps/chosen": -421.11431884765625,
      "logps/rejected": -440.32244873046875,
      "loss": 0.4788,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.025134162977337837,
      "rewards/margins": -0.013268355280160904,
      "rewards/rejected": -0.011865806765854359,
      "step": 28
    },
    {
      "debug/policy_chosen_logits": 32.95792770385742,
      "debug/policy_chosen_logps": -438.89129638671875,
      "debug/policy_rejected_logits": 32.24089813232422,
      "debug/policy_rejected_logps": -445.365966796875,
      "debug/reference_chosen_logps": -440.4767761230469,
      "debug/reference_rejected_logps": -440.51837158203125,
      "epoch": 0.6041666666666666,
      "grad_norm": 4.632495681187026,
      "learning_rate": 1e-06,
      "logits/chosen": 32.95792770385742,
      "logits/rejected": 32.24089813232422,
      "logps/chosen": -438.89129638671875,
      "logps/rejected": -445.365966796875,
      "loss": 0.4649,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015854567289352417,
      "rewards/margins": 0.06433063000440598,
      "rewards/rejected": -0.04847606271505356,
      "step": 29
    },
    {
      "debug/policy_chosen_logits": 31.374120712280273,
      "debug/policy_chosen_logps": -412.0467529296875,
      "debug/policy_rejected_logits": 30.670644760131836,
      "debug/policy_rejected_logps": -437.80059814453125,
      "debug/reference_chosen_logps": -414.59442138671875,
      "debug/reference_rejected_logps": -433.7508544921875,
      "epoch": 0.625,
      "grad_norm": 5.135137765814023,
      "learning_rate": 1e-06,
      "logits/chosen": 31.374120712280273,
      "logits/rejected": 30.670644760131836,
      "logps/chosen": -412.0467529296875,
      "logps/rejected": -437.80059814453125,
      "loss": 0.4928,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.025476723909378052,
      "rewards/margins": 0.06597384810447693,
      "rewards/rejected": -0.040497127920389175,
      "step": 30
    },
    {
      "debug/policy_chosen_logits": 28.232507705688477,
      "debug/policy_chosen_logps": -414.2076721191406,
      "debug/policy_rejected_logits": 27.216352462768555,
      "debug/policy_rejected_logps": -437.1291809082031,
      "debug/reference_chosen_logps": -417.13665771484375,
      "debug/reference_rejected_logps": -431.5104064941406,
      "epoch": 0.6458333333333334,
      "grad_norm": 9.586730066675056,
      "learning_rate": 1e-06,
      "logits/chosen": 28.232507705688477,
      "logits/rejected": 27.216352462768555,
      "logps/chosen": -414.2076721191406,
      "logps/rejected": -437.1291809082031,
      "loss": 0.4718,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.029290199279785156,
      "rewards/margins": 0.08547790348529816,
      "rewards/rejected": -0.0561877004802227,
      "step": 31
    },
    {
      "debug/policy_chosen_logits": 31.265499114990234,
      "debug/policy_chosen_logps": -415.2708740234375,
      "debug/policy_rejected_logits": 30.586397171020508,
      "debug/policy_rejected_logps": -420.286376953125,
      "debug/reference_chosen_logps": -415.6477966308594,
      "debug/reference_rejected_logps": -419.3028259277344,
      "epoch": 0.6666666666666666,
      "grad_norm": 5.088803834381108,
      "learning_rate": 1e-06,
      "logits/chosen": 31.265499114990234,
      "logits/rejected": 30.586397171020508,
      "logps/chosen": -415.2708740234375,
      "logps/rejected": -420.286376953125,
      "loss": 0.4781,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003769032657146454,
      "rewards/margins": 0.013604126870632172,
      "rewards/rejected": -0.00983509048819542,
      "step": 32
    },
    {
      "debug/policy_chosen_logits": 28.746450424194336,
      "debug/policy_chosen_logps": -418.0946350097656,
      "debug/policy_rejected_logits": 30.699132919311523,
      "debug/policy_rejected_logps": -422.5447998046875,
      "debug/reference_chosen_logps": -421.5987243652344,
      "debug/reference_rejected_logps": -423.47491455078125,
      "epoch": 0.6875,
      "grad_norm": 5.2883474953854925,
      "learning_rate": 1e-06,
      "logits/chosen": 28.746450424194336,
      "logits/rejected": 30.699132919311523,
      "logps/chosen": -418.0946350097656,
      "logps/rejected": -422.5447998046875,
      "loss": 0.4925,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03504092991352081,
      "rewards/margins": 0.02574004977941513,
      "rewards/rejected": 0.009300878271460533,
      "step": 33
    },
    {
      "debug/policy_chosen_logits": 32.71680450439453,
      "debug/policy_chosen_logps": -405.2165222167969,
      "debug/policy_rejected_logits": 29.466472625732422,
      "debug/policy_rejected_logps": -449.44921875,
      "debug/reference_chosen_logps": -407.29217529296875,
      "debug/reference_rejected_logps": -445.6148681640625,
      "epoch": 0.7083333333333334,
      "grad_norm": 5.307136992049977,
      "learning_rate": 1e-06,
      "logits/chosen": 32.71680450439453,
      "logits/rejected": 29.466472625732422,
      "logps/chosen": -405.2165222167969,
      "logps/rejected": -449.44921875,
      "loss": 0.4894,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.020756645128130913,
      "rewards/margins": 0.05910034105181694,
      "rewards/rejected": -0.03834369778633118,
      "step": 34
    },
    {
      "debug/policy_chosen_logits": 25.695634841918945,
      "debug/policy_chosen_logps": -404.29071044921875,
      "debug/policy_rejected_logits": 27.330564498901367,
      "debug/policy_rejected_logps": -454.2277526855469,
      "debug/reference_chosen_logps": -405.4884338378906,
      "debug/reference_rejected_logps": -449.79266357421875,
      "epoch": 0.7291666666666666,
      "grad_norm": 4.968733221673012,
      "learning_rate": 1e-06,
      "logits/chosen": 25.695634841918945,
      "logits/rejected": 27.330564498901367,
      "logps/chosen": -404.29071044921875,
      "logps/rejected": -454.2277526855469,
      "loss": 0.446,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011977346614003181,
      "rewards/margins": 0.05632827803492546,
      "rewards/rejected": -0.04435092955827713,
      "step": 35
    },
    {
      "debug/policy_chosen_logits": 27.627330780029297,
      "debug/policy_chosen_logps": -420.744873046875,
      "debug/policy_rejected_logits": 28.073627471923828,
      "debug/policy_rejected_logps": -437.855224609375,
      "debug/reference_chosen_logps": -425.2474060058594,
      "debug/reference_rejected_logps": -429.3760681152344,
      "epoch": 0.75,
      "grad_norm": 5.074316911246501,
      "learning_rate": 1e-06,
      "logits/chosen": 27.627330780029297,
      "logits/rejected": 28.073627471923828,
      "logps/chosen": -420.744873046875,
      "logps/rejected": -437.855224609375,
      "loss": 0.4719,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04502537101507187,
      "rewards/margins": 0.12981674075126648,
      "rewards/rejected": -0.08479136973619461,
      "step": 36
    },
    {
      "debug/policy_chosen_logits": 26.058317184448242,
      "debug/policy_chosen_logps": -418.7844543457031,
      "debug/policy_rejected_logits": 27.601564407348633,
      "debug/policy_rejected_logps": -430.4002380371094,
      "debug/reference_chosen_logps": -421.88189697265625,
      "debug/reference_rejected_logps": -429.6895446777344,
      "epoch": 0.7708333333333334,
      "grad_norm": 5.110386744264913,
      "learning_rate": 1e-06,
      "logits/chosen": 26.058317184448242,
      "logits/rejected": 27.601564407348633,
      "logps/chosen": -418.7844543457031,
      "logps/rejected": -430.4002380371094,
      "loss": 0.5027,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.030974578112363815,
      "rewards/margins": 0.03808154910802841,
      "rewards/rejected": -0.0071069709956645966,
      "step": 37
    },
    {
      "debug/policy_chosen_logits": 32.79731369018555,
      "debug/policy_chosen_logps": -430.58941650390625,
      "debug/policy_rejected_logits": 34.05510330200195,
      "debug/policy_rejected_logps": -450.1502685546875,
      "debug/reference_chosen_logps": -432.3368225097656,
      "debug/reference_rejected_logps": -443.3255615234375,
      "epoch": 0.7916666666666666,
      "grad_norm": 4.973892389169695,
      "learning_rate": 1e-06,
      "logits/chosen": 32.79731369018555,
      "logits/rejected": 34.05510330200195,
      "logps/chosen": -430.58941650390625,
      "logps/rejected": -450.1502685546875,
      "loss": 0.4642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.017473945394158363,
      "rewards/margins": 0.08572139590978622,
      "rewards/rejected": -0.06824745237827301,
      "step": 38
    },
    {
      "debug/policy_chosen_logits": 32.02721405029297,
      "debug/policy_chosen_logps": -420.63739013671875,
      "debug/policy_rejected_logits": 32.61655807495117,
      "debug/policy_rejected_logps": -422.09625244140625,
      "debug/reference_chosen_logps": -425.75048828125,
      "debug/reference_rejected_logps": -412.3810729980469,
      "epoch": 0.8125,
      "grad_norm": 5.161463337049576,
      "learning_rate": 1e-06,
      "logits/chosen": 32.02721405029297,
      "logits/rejected": 32.61655807495117,
      "logps/chosen": -420.63739013671875,
      "logps/rejected": -422.09625244140625,
      "loss": 0.4611,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0511307530105114,
      "rewards/margins": 0.1482827365398407,
      "rewards/rejected": -0.0971519872546196,
      "step": 39
    },
    {
      "debug/policy_chosen_logits": 30.953702926635742,
      "debug/policy_chosen_logps": -415.52679443359375,
      "debug/policy_rejected_logits": 30.896102905273438,
      "debug/policy_rejected_logps": -432.7928161621094,
      "debug/reference_chosen_logps": -418.53839111328125,
      "debug/reference_rejected_logps": -433.76123046875,
      "epoch": 0.8333333333333334,
      "grad_norm": 5.104987202858757,
      "learning_rate": 1e-06,
      "logits/chosen": 30.953702926635742,
      "logits/rejected": 30.896102905273438,
      "logps/chosen": -415.52679443359375,
      "logps/rejected": -432.7928161621094,
      "loss": 0.4728,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03011600486934185,
      "rewards/margins": 0.020431898534297943,
      "rewards/rejected": 0.009684105403721333,
      "step": 40
    },
    {
      "debug/policy_chosen_logits": 31.386384963989258,
      "debug/policy_chosen_logps": -416.5565185546875,
      "debug/policy_rejected_logits": 30.050193786621094,
      "debug/policy_rejected_logps": -419.0783996582031,
      "debug/reference_chosen_logps": -419.27752685546875,
      "debug/reference_rejected_logps": -417.3736267089844,
      "epoch": 0.8541666666666666,
      "grad_norm": 4.8124075843398275,
      "learning_rate": 1e-06,
      "logits/chosen": 31.386384963989258,
      "logits/rejected": 30.050193786621094,
      "logps/chosen": -416.5565185546875,
      "logps/rejected": -419.0783996582031,
      "loss": 0.4822,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.027210043743252754,
      "rewards/margins": 0.04425777494907379,
      "rewards/rejected": -0.017047729343175888,
      "step": 41
    },
    {
      "debug/policy_chosen_logits": 27.5343074798584,
      "debug/policy_chosen_logps": -416.6922607421875,
      "debug/policy_rejected_logits": 29.844606399536133,
      "debug/policy_rejected_logps": -427.3104553222656,
      "debug/reference_chosen_logps": -422.4092102050781,
      "debug/reference_rejected_logps": -425.1396484375,
      "epoch": 0.875,
      "grad_norm": 5.155061770516332,
      "learning_rate": 1e-06,
      "logits/chosen": 27.5343074798584,
      "logits/rejected": 29.844606399536133,
      "logps/chosen": -416.6922607421875,
      "logps/rejected": -427.3104553222656,
      "loss": 0.4577,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05716957151889801,
      "rewards/margins": 0.07887779176235199,
      "rewards/rejected": -0.02170822024345398,
      "step": 42
    },
    {
      "debug/policy_chosen_logits": 27.6116943359375,
      "debug/policy_chosen_logps": -421.576416015625,
      "debug/policy_rejected_logits": 26.841312408447266,
      "debug/policy_rejected_logps": -442.26361083984375,
      "debug/reference_chosen_logps": -428.953125,
      "debug/reference_rejected_logps": -441.0887756347656,
      "epoch": 0.8958333333333334,
      "grad_norm": 4.824458757028807,
      "learning_rate": 1e-06,
      "logits/chosen": 27.6116943359375,
      "logits/rejected": 26.841312408447266,
      "logps/chosen": -421.576416015625,
      "logps/rejected": -442.26361083984375,
      "loss": 0.4393,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07376720011234283,
      "rewards/margins": 0.08551543951034546,
      "rewards/rejected": -0.01174823846668005,
      "step": 43
    },
    {
      "debug/policy_chosen_logits": 28.251888275146484,
      "debug/policy_chosen_logps": -413.2276611328125,
      "debug/policy_rejected_logits": 27.731847763061523,
      "debug/policy_rejected_logps": -431.4485168457031,
      "debug/reference_chosen_logps": -421.6210632324219,
      "debug/reference_rejected_logps": -428.9674072265625,
      "epoch": 0.9166666666666666,
      "grad_norm": 4.765787027247756,
      "learning_rate": 1e-06,
      "logits/chosen": 28.251888275146484,
      "logits/rejected": 27.731847763061523,
      "logps/chosen": -413.2276611328125,
      "logps/rejected": -431.4485168457031,
      "loss": 0.4476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08393420279026031,
      "rewards/margins": 0.10874545574188232,
      "rewards/rejected": -0.02481124736368656,
      "step": 44
    },
    {
      "debug/policy_chosen_logits": 27.165687561035156,
      "debug/policy_chosen_logps": -416.60101318359375,
      "debug/policy_rejected_logits": 26.6442928314209,
      "debug/policy_rejected_logps": -421.559326171875,
      "debug/reference_chosen_logps": -422.03057861328125,
      "debug/reference_rejected_logps": -421.613037109375,
      "epoch": 0.9375,
      "grad_norm": 5.601545923208478,
      "learning_rate": 1e-06,
      "logits/chosen": 27.165687561035156,
      "logits/rejected": 26.6442928314209,
      "logps/chosen": -416.60101318359375,
      "logps/rejected": -421.559326171875,
      "loss": 0.4536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05429550260305405,
      "rewards/margins": 0.05375823751091957,
      "rewards/rejected": 0.0005372613668441772,
      "step": 45
    },
    {
      "debug/policy_chosen_logits": 30.84551429748535,
      "debug/policy_chosen_logps": -420.135498046875,
      "debug/policy_rejected_logits": 31.129411697387695,
      "debug/policy_rejected_logps": -423.7474365234375,
      "debug/reference_chosen_logps": -425.0477294921875,
      "debug/reference_rejected_logps": -426.3204345703125,
      "epoch": 0.9583333333333334,
      "grad_norm": 5.0077997901080655,
      "learning_rate": 1e-06,
      "logits/chosen": 30.84551429748535,
      "logits/rejected": 31.129411697387695,
      "logps/chosen": -420.135498046875,
      "logps/rejected": -423.7474365234375,
      "loss": 0.4604,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04912235215306282,
      "rewards/margins": 0.02339225634932518,
      "rewards/rejected": 0.025730092078447342,
      "step": 46
    },
    {
      "debug/policy_chosen_logits": 25.565441131591797,
      "debug/policy_chosen_logps": -390.5962829589844,
      "debug/policy_rejected_logits": 31.802610397338867,
      "debug/policy_rejected_logps": -443.27691650390625,
      "debug/reference_chosen_logps": -394.64532470703125,
      "debug/reference_rejected_logps": -439.7212219238281,
      "epoch": 0.9791666666666666,
      "grad_norm": 5.104824777439459,
      "learning_rate": 1e-06,
      "logits/chosen": 25.565441131591797,
      "logits/rejected": 31.802610397338867,
      "logps/chosen": -390.5962829589844,
      "logps/rejected": -443.27691650390625,
      "loss": 0.4627,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04049030318856239,
      "rewards/margins": 0.07604698091745377,
      "rewards/rejected": -0.03555667772889137,
      "step": 47
    },
    {
      "debug/policy_chosen_logits": 30.14870262145996,
      "debug/policy_chosen_logps": -410.57965087890625,
      "debug/policy_rejected_logits": 30.24129295349121,
      "debug/policy_rejected_logps": -411.0328369140625,
      "debug/reference_chosen_logps": -416.5841979980469,
      "debug/reference_rejected_logps": -415.076904296875,
      "epoch": 1.0,
      "grad_norm": 5.41196633275443,
      "learning_rate": 1e-06,
      "logits/chosen": 30.14870262145996,
      "logits/rejected": 30.24129295349121,
      "logps/chosen": -410.57965087890625,
      "logps/rejected": -411.0328369140625,
      "loss": 0.4479,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.06004543602466583,
      "rewards/margins": 0.01960449479520321,
      "rewards/rejected": 0.040440939366817474,
      "step": 48
    },
    {
      "epoch": 1.0,
      "step": 48,
      "total_flos": 0.0,
      "train_loss": 0.4809455207238595,
      "train_runtime": 486.3354,
      "train_samples_per_second": 6.271,
      "train_steps_per_second": 0.099
    }
  ],
  "logging_steps": 1,
  "max_steps": 48,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}