{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 1250,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 4e-08,
      "logits/chosen": -3.1403026580810547,
      "logits/rejected": -2.4295239448547363,
      "logps/chosen": -392.2955017089844,
      "logps/rejected": -613.5740966796875,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.0000000000000003e-07,
      "logits/chosen": -2.442929267883301,
      "logits/rejected": -2.321220636367798,
      "logps/chosen": -261.98870849609375,
      "logps/rejected": -221.4100799560547,
      "loss": 0.6931,
      "rewards/accuracies": 0.4722222089767456,
      "rewards/chosen": 7.1744441811461e-05,
      "rewards/margins": 5.5226653785211965e-05,
      "rewards/rejected": 1.6517798940185457e-05,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.000000000000001e-07,
      "logits/chosen": -2.42722225189209,
      "logits/rejected": -2.285809278488159,
      "logps/chosen": -244.7586212158203,
      "logps/rejected": -254.1322784423828,
      "loss": 0.6931,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -4.93793522764463e-05,
      "rewards/margins": 0.00013055391900707036,
      "rewards/rejected": -0.00017993326764553785,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.2000000000000002e-06,
      "logits/chosen": -2.468625068664551,
      "logits/rejected": -2.3896145820617676,
      "logps/chosen": -251.3707733154297,
      "logps/rejected": -300.4630126953125,
      "loss": 0.6931,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.00025913942954503,
      "rewards/margins": 9.763217531144619e-05,
      "rewards/rejected": 0.00016150726878549904,
      "step": 30
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6000000000000001e-06,
      "logits/chosen": -2.483097553253174,
      "logits/rejected": -2.135282516479492,
      "logps/chosen": -252.2518310546875,
      "logps/rejected": -206.49740600585938,
      "loss": 0.693,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0007439021137543023,
      "rewards/margins": 0.0003426038019824773,
      "rewards/rejected": 0.00040129836997948587,
      "step": 40
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0000000000000003e-06,
      "logits/chosen": -2.6477389335632324,
      "logits/rejected": -2.486539363861084,
      "logps/chosen": -245.1098175048828,
      "logps/rejected": -234.5915985107422,
      "loss": 0.6928,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0010015963343903422,
      "rewards/margins": 0.00047645685845054686,
      "rewards/rejected": 0.0005251394468359649,
      "step": 50
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4000000000000003e-06,
      "logits/chosen": -2.5867533683776855,
      "logits/rejected": -1.8461745977401733,
      "logps/chosen": -288.46343994140625,
      "logps/rejected": -191.20741271972656,
      "loss": 0.6924,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0025676097720861435,
      "rewards/margins": 0.001473221811465919,
      "rewards/rejected": 0.0010943879606202245,
      "step": 60
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.8000000000000003e-06,
      "logits/chosen": -2.4274649620056152,
      "logits/rejected": -2.21500825881958,
      "logps/chosen": -206.21188354492188,
      "logps/rejected": -188.66172790527344,
      "loss": 0.6918,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.004373534582555294,
      "rewards/margins": 0.0019022986525669694,
      "rewards/rejected": 0.0024712360464036465,
      "step": 70
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.2000000000000003e-06,
      "logits/chosen": -2.272698163986206,
      "logits/rejected": -2.2475826740264893,
      "logps/chosen": -218.1708984375,
      "logps/rejected": -247.9695281982422,
      "loss": 0.6919,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.00605297926813364,
      "rewards/margins": 0.002590904710814357,
      "rewards/rejected": 0.0034620738588273525,
      "step": 80
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.6000000000000003e-06,
      "logits/chosen": -2.5878186225891113,
      "logits/rejected": -2.174198627471924,
      "logps/chosen": -284.7269287109375,
      "logps/rejected": -222.1537322998047,
      "loss": 0.6908,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.00458324421197176,
      "rewards/margins": 0.005124131683260202,
      "rewards/rejected": -0.0005408872966654599,
      "step": 90
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.000000000000001e-06,
      "logits/chosen": -2.4876809120178223,
      "logits/rejected": -2.304762363433838,
      "logps/chosen": -258.6788330078125,
      "logps/rejected": -230.6868133544922,
      "loss": 0.6878,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0009092213585972786,
      "rewards/margins": 0.013424595817923546,
      "rewards/rejected": -0.012515375390648842,
      "step": 100
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.4e-06,
      "logits/chosen": -2.24761700630188,
      "logits/rejected": -2.0283989906311035,
      "logps/chosen": -242.88485717773438,
      "logps/rejected": -253.8578643798828,
      "loss": 0.6832,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02561577782034874,
      "rewards/margins": 0.02318538725376129,
      "rewards/rejected": -0.04880116507411003,
      "step": 110
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.800000000000001e-06,
      "logits/chosen": -2.0988376140594482,
      "logits/rejected": -2.193089008331299,
      "logps/chosen": -242.8556365966797,
      "logps/rejected": -317.71917724609375,
      "loss": 0.6753,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.047444917261600494,
      "rewards/margins": 0.03924765810370445,
      "rewards/rejected": -0.08669257164001465,
      "step": 120
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999756310023261e-06,
      "logits/chosen": -2.241037607192993,
      "logits/rejected": -2.157954692840576,
      "logps/chosen": -331.423828125,
      "logps/rejected": -348.91046142578125,
      "loss": 0.6688,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08836604654788971,
      "rewards/margins": 0.0334697850048542,
      "rewards/rejected": -0.12183582782745361,
      "step": 130
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997807075247147e-06,
      "logits/chosen": -1.7352321147918701,
      "logits/rejected": -1.5876859426498413,
      "logps/chosen": -398.1210021972656,
      "logps/rejected": -428.30328369140625,
      "loss": 0.6519,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.22779341042041779,
      "rewards/margins": 0.0445188470184803,
      "rewards/rejected": -0.2723122239112854,
      "step": 140
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.993910125649561e-06,
      "logits/chosen": -1.735691785812378,
      "logits/rejected": -1.7642030715942383,
      "logps/chosen": -525.7499389648438,
      "logps/rejected": -695.8289184570312,
      "loss": 0.6354,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.30290618538856506,
      "rewards/margins": 0.11975699663162231,
      "rewards/rejected": -0.422663152217865,
      "step": 150
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.988068499954578e-06,
      "logits/chosen": -2.004002332687378,
      "logits/rejected": -1.7766345739364624,
      "logps/chosen": -464.7952575683594,
      "logps/rejected": -630.8657836914062,
      "loss": 0.6461,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23786354064941406,
      "rewards/margins": 0.18317463994026184,
      "rewards/rejected": -0.4210381507873535,
      "step": 160
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980286753286196e-06,
      "logits/chosen": -2.0650768280029297,
      "logits/rejected": -1.572537899017334,
      "logps/chosen": -591.57421875,
      "logps/rejected": -780.8646240234375,
      "loss": 0.607,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2891489565372467,
      "rewards/margins": 0.27657198905944824,
      "rewards/rejected": -0.5657209157943726,
      "step": 170
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.970570953616383e-06,
      "logits/chosen": -1.4938254356384277,
      "logits/rejected": -1.661007285118103,
      "logps/chosen": -764.5491943359375,
      "logps/rejected": -1000.9505615234375,
      "loss": 0.6384,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5677521228790283,
      "rewards/margins": 0.19688071310520172,
      "rewards/rejected": -0.7646328806877136,
      "step": 180
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.958928677033465e-06,
      "logits/chosen": -1.6295101642608643,
      "logits/rejected": -1.3617414236068726,
      "logps/chosen": -720.4210815429688,
      "logps/rejected": -925.54638671875,
      "loss": 0.6042,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4372948706150055,
      "rewards/margins": 0.2413189709186554,
      "rewards/rejected": -0.6786138415336609,
      "step": 190
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9453690018345144e-06,
      "logits/chosen": -1.502726674079895,
      "logits/rejected": -1.3334269523620605,
      "logps/chosen": -752.0069580078125,
      "logps/rejected": -1068.231201171875,
      "loss": 0.64,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5011454820632935,
      "rewards/margins": 0.2965936064720154,
      "rewards/rejected": -0.7977390289306641,
      "step": 200
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9299025014463665e-06,
      "logits/chosen": -1.1884076595306396,
      "logits/rejected": -1.0619341135025024,
      "logps/chosen": -890.1341552734375,
      "logps/rejected": -1238.742431640625,
      "loss": 0.6007,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6316002607345581,
      "rewards/margins": 0.3692142069339752,
      "rewards/rejected": -1.000814437866211,
      "step": 210
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.912541236180779e-06,
      "logits/chosen": -1.0146244764328003,
      "logits/rejected": -0.9316056966781616,
      "logps/chosen": -979.2394409179688,
      "logps/rejected": -1465.3270263671875,
      "loss": 0.6018,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7414819002151489,
      "rewards/margins": 0.49276018142700195,
      "rewards/rejected": -1.2342422008514404,
      "step": 220
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.893298743830168e-06,
      "logits/chosen": -1.2160202264785767,
      "logits/rejected": -1.0113346576690674,
      "logps/chosen": -1194.5838623046875,
      "logps/rejected": -1710.5823974609375,
      "loss": 0.5866,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9506030082702637,
      "rewards/margins": 0.5406419634819031,
      "rewards/rejected": -1.491244912147522,
      "step": 230
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8721900291112415e-06,
      "logits/chosen": -1.7798312902450562,
      "logits/rejected": -1.6643425226211548,
      "logps/chosen": -1057.5977783203125,
      "logps/rejected": -1005.8741455078125,
      "loss": 0.694,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7734619379043579,
      "rewards/margins": -0.0247958954423666,
      "rewards/rejected": -0.7486661076545715,
      "step": 240
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.849231551964771e-06,
      "logits/chosen": -2.0089643001556396,
      "logits/rejected": -1.7859022617340088,
      "logps/chosen": -399.1302795410156,
      "logps/rejected": -577.6984252929688,
      "loss": 0.6424,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1754738688468933,
      "rewards/margins": 0.13275322318077087,
      "rewards/rejected": -0.3082270920276642,
      "step": 250
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.824441214720629e-06,
      "logits/chosen": -1.7344192266464233,
      "logits/rejected": -1.632328748703003,
      "logps/chosen": -632.4493408203125,
      "logps/rejected": -760.2273559570312,
      "loss": 0.6252,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.41359105706214905,
      "rewards/margins": 0.12629616260528564,
      "rewards/rejected": -0.5398872494697571,
      "step": 260
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7978383481380865e-06,
      "logits/chosen": -1.5548350811004639,
      "logits/rejected": -1.5831079483032227,
      "logps/chosen": -598.45556640625,
      "logps/rejected": -712.7095336914062,
      "loss": 0.6096,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3854503631591797,
      "rewards/margins": 0.13576054573059082,
      "rewards/rejected": -0.5212109088897705,
      "step": 270
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.769443696332272e-06,
      "logits/chosen": -2.0036838054656982,
      "logits/rejected": -1.7330118417739868,
      "logps/chosen": -426.128662109375,
      "logps/rejected": -617.08984375,
      "loss": 0.6208,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2103346288204193,
      "rewards/margins": 0.19718368351459503,
      "rewards/rejected": -0.40751829743385315,
      "step": 280
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7392794005985324e-06,
      "logits/chosen": -1.8511890172958374,
      "logits/rejected": -1.5459794998168945,
      "logps/chosen": -737.9730224609375,
      "logps/rejected": -961.6851806640625,
      "loss": 0.6214,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.48743873834609985,
      "rewards/margins": 0.20717349648475647,
      "rewards/rejected": -0.6946121454238892,
      "step": 290
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.707368982147318e-06,
      "logits/chosen": -1.7424571514129639,
      "logits/rejected": -1.4730459451675415,
      "logps/chosen": -833.6031494140625,
      "logps/rejected": -950.7183837890625,
      "loss": 0.5805,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5092605352401733,
      "rewards/margins": 0.1874401867389679,
      "rewards/rejected": -0.6967006921768188,
      "step": 300
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.673737323763048e-06,
      "logits/chosen": -1.373366355895996,
      "logits/rejected": -1.2923896312713623,
      "logps/chosen": -975.0471801757812,
      "logps/rejected": -1144.96533203125,
      "loss": 0.6091,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.7069741487503052,
      "rewards/margins": 0.21132870018482208,
      "rewards/rejected": -0.9183028340339661,
      "step": 310
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.638410650401267e-06,
      "logits/chosen": -1.477074384689331,
      "logits/rejected": -1.2639570236206055,
      "logps/chosen": -1050.070556640625,
      "logps/rejected": -1336.6949462890625,
      "loss": 0.5927,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8159879446029663,
      "rewards/margins": 0.2922418415546417,
      "rewards/rejected": -1.1082298755645752,
      "step": 320
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.601416508739211e-06,
      "logits/chosen": -1.5806219577789307,
      "logits/rejected": -1.5797499418258667,
      "logps/chosen": -1069.904296875,
      "logps/rejected": -1329.1131591796875,
      "loss": 0.6329,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.8489059209823608,
      "rewards/margins": 0.2538915276527405,
      "rewards/rejected": -1.102797508239746,
      "step": 330
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.562783745695738e-06,
      "logits/chosen": -1.578185796737671,
      "logits/rejected": -1.3778866529464722,
      "logps/chosen": -898.2579956054688,
      "logps/rejected": -1295.5048828125,
      "loss": 0.5816,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6616091728210449,
      "rewards/margins": 0.3867344856262207,
      "rewards/rejected": -1.0483437776565552,
      "step": 340
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.522542485937369e-06,
      "logits/chosen": -1.3196460008621216,
      "logits/rejected": -1.0029346942901611,
      "logps/chosen": -1030.9923095703125,
      "logps/rejected": -1271.06494140625,
      "loss": 0.6062,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7833272218704224,
      "rewards/margins": 0.2811380922794342,
      "rewards/rejected": -1.0644652843475342,
      "step": 350
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4807241083879774e-06,
      "logits/chosen": -1.4315626621246338,
      "logits/rejected": -1.1289265155792236,
      "logps/chosen": -1395.56640625,
      "logps/rejected": -2238.73681640625,
      "loss": 0.5686,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.1033650636672974,
      "rewards/margins": 0.8617695569992065,
      "rewards/rejected": -1.965134620666504,
      "step": 360
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.437361221760449e-06,
      "logits/chosen": -1.0882983207702637,
      "logits/rejected": -0.9719398617744446,
      "logps/chosen": -1259.433349609375,
      "logps/rejected": -1656.2109375,
      "loss": 0.5973,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.069411277770996,
      "rewards/margins": 0.3754344880580902,
      "rewards/rejected": -1.4448457956314087,
      "step": 370
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.3924876391293915e-06,
      "logits/chosen": -0.9243530035018921,
      "logits/rejected": -0.9347097277641296,
      "logps/chosen": -1276.580810546875,
      "logps/rejected": -1660.425048828125,
      "loss": 0.5932,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.1309508085250854,
      "rewards/margins": 0.35707855224609375,
      "rewards/rejected": -1.4880293607711792,
      "step": 380
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.346138351564711e-06,
      "logits/chosen": -1.2673766613006592,
      "logits/rejected": -1.1589628458023071,
      "logps/chosen": -1206.547607421875,
      "logps/rejected": -1615.82568359375,
      "loss": 0.5588,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9714127779006958,
      "rewards/margins": 0.4280439019203186,
      "rewards/rejected": -1.39945650100708,
      "step": 390
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.2983495008466285e-06,
      "logits/chosen": -1.5774317979812622,
      "logits/rejected": -1.2660281658172607,
      "logps/chosen": -1082.2581787109375,
      "logps/rejected": -1570.060791015625,
      "loss": 0.5484,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8231463432312012,
      "rewards/margins": 0.5096359848976135,
      "rewards/rejected": -1.3327823877334595,
      "step": 400
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.249158351283414e-06,
      "logits/chosen": -1.4447729587554932,
      "logits/rejected": -1.1427133083343506,
      "logps/chosen": -1174.7425537109375,
      "logps/rejected": -1466.5479736328125,
      "loss": 0.5432,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9159573316574097,
      "rewards/margins": 0.33625325560569763,
      "rewards/rejected": -1.2522106170654297,
      "step": 410
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.198603260653792e-06,
      "logits/chosen": -0.7784031629562378,
      "logits/rejected": -0.5429641008377075,
      "logps/chosen": -2375.2294921875,
      "logps/rejected": -2860.4833984375,
      "loss": 0.5023,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.1455845832824707,
      "rewards/margins": 0.4987269341945648,
      "rewards/rejected": -2.6443114280700684,
      "step": 420
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.146723650296701e-06,
      "logits/chosen": -0.8310529589653015,
      "logits/rejected": -0.47391629219055176,
      "logps/chosen": -3060.8125,
      "logps/rejected": -3744.44140625,
      "loss": 0.6076,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.840507984161377,
      "rewards/margins": 0.725513219833374,
      "rewards/rejected": -3.56602144241333,
      "step": 430
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.093559974371725e-06,
      "logits/chosen": -1.8024406433105469,
      "logits/rejected": -1.4066709280014038,
      "logps/chosen": -1206.559326171875,
      "logps/rejected": -1958.577392578125,
      "loss": 0.5517,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.948940098285675,
      "rewards/margins": 0.76356440782547,
      "rewards/rejected": -1.7125043869018555,
      "step": 440
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.039153688314146e-06,
      "logits/chosen": -1.7638174295425415,
      "logits/rejected": -1.6592317819595337,
      "logps/chosen": -888.7286376953125,
      "logps/rejected": -1274.7813720703125,
      "loss": 0.5681,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6799342036247253,
      "rewards/margins": 0.4119555950164795,
      "rewards/rejected": -1.0918899774551392,
      "step": 450
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.983547216509254e-06,
      "logits/chosen": -1.8817472457885742,
      "logits/rejected": -1.766122817993164,
      "logps/chosen": -1327.4205322265625,
      "logps/rejected": -1799.326904296875,
      "loss": 0.5546,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9942048192024231,
      "rewards/margins": 0.49855145812034607,
      "rewards/rejected": -1.4927562475204468,
      "step": 460
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.92678391921108e-06,
      "logits/chosen": -1.842795968055725,
      "logits/rejected": -1.6158469915390015,
      "logps/chosen": -1413.307373046875,
      "logps/rejected": -1754.1558837890625,
      "loss": 0.6049,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0906217098236084,
      "rewards/margins": 0.3714645802974701,
      "rewards/rejected": -1.4620864391326904,
      "step": 470
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.868908058731376e-06,
      "logits/chosen": -1.6520286798477173,
      "logits/rejected": -1.5457327365875244,
      "logps/chosen": -1587.302490234375,
      "logps/rejected": -1965.464599609375,
      "loss": 0.5766,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3615858554840088,
      "rewards/margins": 0.39060306549072266,
      "rewards/rejected": -1.7521886825561523,
      "step": 480
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8099647649251984e-06,
      "logits/chosen": -1.8340473175048828,
      "logits/rejected": -1.6425994634628296,
      "logps/chosen": -1667.3792724609375,
      "logps/rejected": -2022.701904296875,
      "loss": 0.5243,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4147528409957886,
      "rewards/margins": 0.4026387333869934,
      "rewards/rejected": -1.8173916339874268,
      "step": 490
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": -1.565735936164856,
      "logits/rejected": -1.4348723888397217,
      "logps/chosen": -1869.2935791015625,
      "logps/rejected": -2379.262939453125,
      "loss": 0.5181,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.658975601196289,
      "rewards/margins": 0.5309830904006958,
      "rewards/rejected": -2.1899585723876953,
      "step": 500
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.689060522675689e-06,
      "logits/chosen": -1.8067477941513062,
      "logits/rejected": -1.4334272146224976,
      "logps/chosen": -1798.0201416015625,
      "logps/rejected": -2005.1083984375,
      "loss": 0.5859,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.5768072605133057,
      "rewards/margins": 0.24011941254138947,
      "rewards/rejected": -1.8169265985488892,
      "step": 510
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.627193851723577e-06,
      "logits/chosen": -1.9058891534805298,
      "logits/rejected": -1.5708621740341187,
      "logps/chosen": -1344.331787109375,
      "logps/rejected": -2066.19677734375,
      "loss": 0.5302,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.0564631223678589,
      "rewards/margins": 0.7303067445755005,
      "rewards/rejected": -1.7867698669433594,
      "step": 520
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.564448228912682e-06,
      "logits/chosen": -1.8703407049179077,
      "logits/rejected": -1.8959243297576904,
      "logps/chosen": -1418.962890625,
      "logps/rejected": -1918.7298583984375,
      "loss": 0.5437,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.2395734786987305,
      "rewards/margins": 0.4607962965965271,
      "rewards/rejected": -1.7003698348999023,
      "step": 530
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5008725813922383e-06,
      "logits/chosen": -1.4778034687042236,
      "logits/rejected": -1.42378830909729,
      "logps/chosen": -2291.814453125,
      "logps/rejected": -2982.93212890625,
      "loss": 0.4967,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.091310977935791,
      "rewards/margins": 0.6848040819168091,
      "rewards/rejected": -2.7761149406433105,
      "step": 540
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.436516483539781e-06,
      "logits/chosen": -1.495721459388733,
      "logits/rejected": -1.364970326423645,
      "logps/chosen": -2159.25634765625,
      "logps/rejected": -2941.31982421875,
      "loss": 0.5244,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9415247440338135,
      "rewards/margins": 0.7436372637748718,
      "rewards/rejected": -2.685161828994751,
      "step": 550
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3714301183045382e-06,
      "logits/chosen": -1.7252031564712524,
      "logits/rejected": -1.3431731462478638,
      "logps/chosen": -2346.15673828125,
      "logps/rejected": -2880.50341796875,
      "loss": 0.5818,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.072269916534424,
      "rewards/margins": 0.5818148851394653,
      "rewards/rejected": -2.6540849208831787,
      "step": 560
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.3056642380762783e-06,
      "logits/chosen": -1.7475759983062744,
      "logits/rejected": -1.6887413263320923,
      "logps/chosen": -2365.16650390625,
      "logps/rejected": -3232.527587890625,
      "loss": 0.4838,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.112750291824341,
      "rewards/margins": 0.815180778503418,
      "rewards/rejected": -2.927931070327759,
      "step": 570
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2392701251101172e-06,
      "logits/chosen": -1.6227779388427734,
      "logits/rejected": -1.371058702468872,
      "logps/chosen": -2093.24365234375,
      "logps/rejected": -2805.071533203125,
      "loss": 0.5206,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.8952558040618896,
      "rewards/margins": 0.6814407706260681,
      "rewards/rejected": -2.5766968727111816,
      "step": 580
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1722995515381644e-06,
      "logits/chosen": -1.6335232257843018,
      "logits/rejected": -1.2477694749832153,
      "logps/chosen": -2436.821044921875,
      "logps/rejected": -3250.9990234375,
      "loss": 0.5081,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2077012062072754,
      "rewards/margins": 0.8523558378219604,
      "rewards/rejected": -3.0600569248199463,
      "step": 590
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1048047389991693e-06,
      "logits/chosen": -1.898289442062378,
      "logits/rejected": -1.5713775157928467,
      "logps/chosen": -1884.5660400390625,
      "logps/rejected": -2746.454833984375,
      "loss": 0.5623,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6871427297592163,
      "rewards/margins": 0.8711158633232117,
      "rewards/rejected": -2.5582587718963623,
      "step": 600
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0368383179176584e-06,
      "logits/chosen": -1.4297007322311401,
      "logits/rejected": -1.3948471546173096,
      "logps/chosen": -2035.6175537109375,
      "logps/rejected": -3012.0087890625,
      "loss": 0.5395,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.860030174255371,
      "rewards/margins": 0.9378849267959595,
      "rewards/rejected": -2.797914981842041,
      "step": 610
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9684532864643123e-06,
      "logits/chosen": -1.7777831554412842,
      "logits/rejected": -1.3341039419174194,
      "logps/chosen": -2097.790283203125,
      "logps/rejected": -2818.270263671875,
      "loss": 0.4943,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8172571659088135,
      "rewards/margins": 0.7645599246025085,
      "rewards/rejected": -2.581817150115967,
      "step": 620
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.8997029692295875e-06,
      "logits/chosen": -1.7123359441757202,
      "logits/rejected": -1.4930963516235352,
      "logps/chosen": -1850.494873046875,
      "logps/rejected": -2223.23779296875,
      "loss": 0.5415,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5890159606933594,
      "rewards/margins": 0.4220260679721832,
      "rewards/rejected": -2.0110421180725098,
      "step": 630
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8306409756428067e-06,
      "logits/chosen": -1.7620747089385986,
      "logits/rejected": -1.273874282836914,
      "logps/chosen": -1751.422119140625,
      "logps/rejected": -2349.684814453125,
      "loss": 0.5267,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5236599445343018,
      "rewards/margins": 0.6361467838287354,
      "rewards/rejected": -2.159806728363037,
      "step": 640
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.761321158169134e-06,
      "logits/chosen": -1.8131462335586548,
      "logits/rejected": -1.548789381980896,
      "logps/chosen": -1939.831298828125,
      "logps/rejected": -2459.304443359375,
      "loss": 0.5526,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7154333591461182,
      "rewards/margins": 0.5310146808624268,
      "rewards/rejected": -2.246448040008545,
      "step": 650
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6917975703170466e-06,
      "logits/chosen": -1.4764631986618042,
      "logits/rejected": -1.2985259294509888,
      "logps/chosen": -2074.831787109375,
      "logps/rejected": -2988.404296875,
      "loss": 0.5177,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8811687231063843,
      "rewards/margins": 0.8929063677787781,
      "rewards/rejected": -2.7740752696990967,
      "step": 660
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6221244244890336e-06,
      "logits/chosen": -1.606785774230957,
      "logits/rejected": -1.4506292343139648,
      "logps/chosen": -2944.038330078125,
      "logps/rejected": -3770.83203125,
      "loss": 0.5202,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.648715019226074,
      "rewards/margins": 0.8418272733688354,
      "rewards/rejected": -3.4905428886413574,
      "step": 670
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5523560497083927e-06,
      "logits/chosen": -1.7594387531280518,
      "logits/rejected": -1.4435147047042847,
      "logps/chosen": -2691.794921875,
      "logps/rejected": -2907.418212890625,
      "loss": 0.6347,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.4359123706817627,
      "rewards/margins": 0.3157985210418701,
      "rewards/rejected": -2.751710891723633,
      "step": 680
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.482546849255096e-06,
      "logits/chosen": -1.6512796878814697,
      "logits/rejected": -1.2861438989639282,
      "logps/chosen": -2340.228515625,
      "logps/rejected": -3065.31591796875,
      "loss": 0.4919,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.0771026611328125,
      "rewards/margins": 0.6874440908432007,
      "rewards/rejected": -2.7645466327667236,
      "step": 690
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4127512582437486e-06,
      "logits/chosen": -1.7903190851211548,
      "logits/rejected": -1.5637561082839966,
      "logps/chosen": -1961.098876953125,
      "logps/rejected": -2661.05126953125,
      "loss": 0.5352,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.727064847946167,
      "rewards/margins": 0.7038751840591431,
      "rewards/rejected": -2.4309401512145996,
      "step": 700
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3430237011767166e-06,
      "logits/chosen": -1.5963449478149414,
      "logits/rejected": -1.5752345323562622,
      "logps/chosen": -2412.08447265625,
      "logps/rejected": -2860.287109375,
      "loss": 0.5364,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.1989083290100098,
      "rewards/margins": 0.4225144386291504,
      "rewards/rejected": -2.62142276763916,
      "step": 710
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2734185495055503e-06,
      "logits/chosen": -1.581274390220642,
      "logits/rejected": -1.338945746421814,
      "logps/chosen": -2689.8212890625,
      "logps/rejected": -3143.3369140625,
      "loss": 0.5331,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.4156579971313477,
      "rewards/margins": 0.4918283522129059,
      "rewards/rejected": -2.9074864387512207,
      "step": 720
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2039900792337477e-06,
      "logits/chosen": -1.6093494892120361,
      "logits/rejected": -1.2763807773590088,
      "logps/chosen": -1861.92578125,
      "logps/rejected": -2657.68212890625,
      "loss": 0.4712,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6057929992675781,
      "rewards/margins": 0.847273051738739,
      "rewards/rejected": -2.453066110610962,
      "step": 730
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.134792428593971e-06,
      "logits/chosen": -1.3526313304901123,
      "logits/rejected": -1.3224549293518066,
      "logps/chosen": -2195.27197265625,
      "logps/rejected": -2851.3291015625,
      "loss": 0.5536,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.023761749267578,
      "rewards/margins": 0.6121652722358704,
      "rewards/rejected": -2.635927200317383,
      "step": 740
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0658795558326745e-06,
      "logits/chosen": -1.6000654697418213,
      "logits/rejected": -1.4151827096939087,
      "logps/chosen": -2148.247802734375,
      "logps/rejected": -2729.99853515625,
      "loss": 0.5222,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.903270959854126,
      "rewards/margins": 0.5996490716934204,
      "rewards/rejected": -2.5029196739196777,
      "step": 750
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.997305197135089e-06,
      "logits/chosen": -1.8795032501220703,
      "logits/rejected": -1.8359801769256592,
      "logps/chosen": -2000.689208984375,
      "logps/rejected": -2926.07275390625,
      "loss": 0.5271,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7533314228057861,
      "rewards/margins": 0.9173868298530579,
      "rewards/rejected": -2.6707184314727783,
      "step": 760
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9291228247233607e-06,
      "logits/chosen": -1.6307891607284546,
      "logits/rejected": -1.496146321296692,
      "logps/chosen": -1922.004150390625,
      "logps/rejected": -2854.74365234375,
      "loss": 0.4909,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7167818546295166,
      "rewards/margins": 0.9143074750900269,
      "rewards/rejected": -2.631089448928833,
      "step": 770
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8613856051605242e-06,
      "logits/chosen": -1.5307934284210205,
      "logits/rejected": -1.4985663890838623,
      "logps/chosen": -1941.616455078125,
      "logps/rejected": -2720.62109375,
      "loss": 0.495,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7477505207061768,
      "rewards/margins": 0.7529212236404419,
      "rewards/rejected": -2.500671863555908,
      "step": 780
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7941463578928088e-06,
      "logits/chosen": -1.6022170782089233,
      "logits/rejected": -1.5732767581939697,
      "logps/chosen": -2065.362548828125,
      "logps/rejected": -2565.924560546875,
      "loss": 0.5311,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8574340343475342,
      "rewards/margins": 0.4975149631500244,
      "rewards/rejected": -2.3549489974975586,
      "step": 790
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7274575140626318e-06,
      "logits/chosen": -1.5523189306259155,
      "logits/rejected": -1.2900121212005615,
      "logps/chosen": -2249.78466796875,
      "logps/rejected": -3071.79541015625,
      "loss": 0.5322,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.0411088466644287,
      "rewards/margins": 0.8322113752365112,
      "rewards/rejected": -2.8733203411102295,
      "step": 800
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.661371075624363e-06,
      "logits/chosen": -1.750195860862732,
      "logits/rejected": -1.4256199598312378,
      "logps/chosen": -2680.12548828125,
      "logps/rejected": -3339.82177734375,
      "loss": 0.5582,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.413327693939209,
      "rewards/margins": 0.6952977180480957,
      "rewards/rejected": -3.108625650405884,
      "step": 810
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5959385747947697e-06,
      "logits/chosen": -1.6673352718353271,
      "logits/rejected": -1.4309087991714478,
      "logps/chosen": -2934.502685546875,
      "logps/rejected": -3557.366455078125,
      "loss": 0.5826,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.6931636333465576,
      "rewards/margins": 0.6061044931411743,
      "rewards/rejected": -3.2992682456970215,
      "step": 820
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5312110338697427e-06,
      "logits/chosen": -1.4265161752700806,
      "logits/rejected": -1.3840219974517822,
      "logps/chosen": -2902.10205078125,
      "logps/rejected": -3211.46484375,
      "loss": 0.5793,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.7132956981658936,
      "rewards/margins": 0.2958894670009613,
      "rewards/rejected": -3.009185314178467,
      "step": 830
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.467238925438646e-06,
      "logits/chosen": -1.93195378780365,
      "logits/rejected": -1.7828657627105713,
      "logps/chosen": -2659.49609375,
      "logps/rejected": -3121.871826171875,
      "loss": 0.5751,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.427194118499756,
      "rewards/margins": 0.47958964109420776,
      "rewards/rejected": -2.9067838191986084,
      "step": 840
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4040721330273063e-06,
      "logits/chosen": -1.6783872842788696,
      "logits/rejected": -1.3621320724487305,
      "logps/chosen": -2476.935791015625,
      "logps/rejected": -3240.056640625,
      "loss": 0.5022,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.256714344024658,
      "rewards/margins": 0.7921364307403564,
      "rewards/rejected": -3.0488505363464355,
      "step": 850
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3417599122003464e-06,
      "logits/chosen": -1.631744146347046,
      "logits/rejected": -1.4881222248077393,
      "logps/chosen": -2611.767333984375,
      "logps/rejected": -3342.803466796875,
      "loss": 0.5239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.361072301864624,
      "rewards/margins": 0.6957711577415466,
      "rewards/rejected": -3.0568435192108154,
      "step": 860
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.280350852153168e-06,
      "logits/chosen": -1.6476237773895264,
      "logits/rejected": -1.2135014533996582,
      "logps/chosen": -2210.89697265625,
      "logps/rejected": -3153.556884765625,
      "loss": 0.49,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.9529176950454712,
      "rewards/margins": 0.9891131520271301,
      "rewards/rejected": -2.942031145095825,
      "step": 870
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2198928378235717e-06,
      "logits/chosen": -1.8185428380966187,
      "logits/rejected": -1.4744179248809814,
      "logps/chosen": -1982.8447265625,
      "logps/rejected": -2986.81591796875,
      "loss": 0.4908,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.720696210861206,
      "rewards/margins": 1.0247818231582642,
      "rewards/rejected": -2.7454781532287598,
      "step": 880
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.160433012552508e-06,
      "logits/chosen": -1.9028065204620361,
      "logits/rejected": -1.5971095561981201,
      "logps/chosen": -1988.5980224609375,
      "logps/rejected": -2843.60302734375,
      "loss": 0.5298,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.695246934890747,
      "rewards/margins": 0.9005265235900879,
      "rewards/rejected": -2.595773458480835,
      "step": 890
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1020177413231334e-06,
      "logits/chosen": -1.7520854473114014,
      "logits/rejected": -1.4778826236724854,
      "logps/chosen": -1867.814453125,
      "logps/rejected": -2722.72900390625,
      "loss": 0.5195,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6399002075195312,
      "rewards/margins": 0.8625130653381348,
      "rewards/rejected": -2.502413272857666,
      "step": 900
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0446925746067768e-06,
      "logits/chosen": -1.6180340051651,
      "logits/rejected": -1.4504057168960571,
      "logps/chosen": -2156.244140625,
      "logps/rejected": -3364.479736328125,
      "loss": 0.4724,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9639689922332764,
      "rewards/margins": 1.1746752262115479,
      "rewards/rejected": -3.138643980026245,
      "step": 910
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.88502212844063e-07,
      "logits/chosen": -1.8366491794586182,
      "logits/rejected": -1.353752851486206,
      "logps/chosen": -1940.987060546875,
      "logps/rejected": -2697.123779296875,
      "loss": 0.506,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.714422583580017,
      "rewards/margins": 0.7562412023544312,
      "rewards/rejected": -2.4706637859344482,
      "step": 920
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.334904715888496e-07,
      "logits/chosen": -1.8136298656463623,
      "logits/rejected": -1.5429127216339111,
      "logps/chosen": -2025.2021484375,
      "logps/rejected": -2842.939453125,
      "loss": 0.5274,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.792414903640747,
      "rewards/margins": 0.8433243036270142,
      "rewards/rejected": -2.63573956489563,
      "step": 930
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.797002473421729e-07,
      "logits/chosen": -1.6221929788589478,
      "logits/rejected": -1.536675214767456,
      "logps/chosen": -2179.556884765625,
      "logps/rejected": -2724.71142578125,
      "loss": 0.5293,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8998562097549438,
      "rewards/margins": 0.5943135023117065,
      "rewards/rejected": -2.4941699504852295,
      "step": 940
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.271734841028553e-07,
      "logits/chosen": -1.599360704421997,
      "logits/rejected": -1.4945180416107178,
      "logps/chosen": -2247.677001953125,
      "logps/rejected": -2832.541015625,
      "loss": 0.5645,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.041861057281494,
      "rewards/margins": 0.5804768800735474,
      "rewards/rejected": -2.622338056564331,
      "step": 950
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.759511406608255e-07,
      "logits/chosen": -1.9042466878890991,
      "logits/rejected": -1.5334478616714478,
      "logps/chosen": -1946.2919921875,
      "logps/rejected": -2741.82861328125,
      "loss": 0.466,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.713287115097046,
      "rewards/margins": 0.8301644325256348,
      "rewards/rejected": -2.5434517860412598,
      "step": 960
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.260731586586983e-07,
      "logits/chosen": -1.5018055438995361,
      "logits/rejected": -1.4937880039215088,
      "logps/chosen": -2611.099609375,
      "logps/rejected": -3151.30810546875,
      "loss": 0.5775,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.3841938972473145,
      "rewards/margins": 0.5148266553878784,
      "rewards/rejected": -2.8990206718444824,
      "step": 970
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.775784314464717e-07,
      "logits/chosen": -1.6764729022979736,
      "logits/rejected": -1.5057140588760376,
      "logps/chosen": -2298.930908203125,
      "logps/rejected": -2712.99169921875,
      "loss": 0.556,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.0648088455200195,
      "rewards/margins": 0.4065318703651428,
      "rewards/rejected": -2.4713406562805176,
      "step": 980
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.305047737536707e-07,
      "logits/chosen": -1.7027795314788818,
      "logits/rejected": -1.3025496006011963,
      "logps/chosen": -2075.1337890625,
      "logps/rejected": -2931.57080078125,
      "loss": 0.4584,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.846689224243164,
      "rewards/margins": 0.8770244717597961,
      "rewards/rejected": -2.7237138748168945,
      "step": 990
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.848888922025553e-07,
      "logits/chosen": -1.8110439777374268,
      "logits/rejected": -1.4326590299606323,
      "logps/chosen": -2325.86474609375,
      "logps/rejected": -3089.902099609375,
      "loss": 0.5249,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.067896604537964,
      "rewards/margins": 0.7900792360305786,
      "rewards/rejected": -2.857975482940674,
      "step": 1000
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.407663566854008e-07,
      "logits/chosen": -1.6088895797729492,
      "logits/rejected": -1.4155261516571045,
      "logps/chosen": -2324.50927734375,
      "logps/rejected": -3341.44970703125,
      "loss": 0.525,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.1127965450286865,
      "rewards/margins": 0.9637571573257446,
      "rewards/rejected": -3.0765535831451416,
      "step": 1010
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.981715726281666e-07,
      "logits/chosen": -1.6132185459136963,
      "logits/rejected": -1.2942755222320557,
      "logps/chosen": -2351.848388671875,
      "logps/rejected": -3291.189453125,
      "loss": 0.5183,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.1332767009735107,
      "rewards/margins": 0.9448688626289368,
      "rewards/rejected": -3.0781455039978027,
      "step": 1020
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.5713775416217884e-07,
      "logits/chosen": -1.770582914352417,
      "logits/rejected": -1.515246868133545,
      "logps/chosen": -2241.62109375,
      "logps/rejected": -3231.28125,
      "loss": 0.4636,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9664819240570068,
      "rewards/margins": 1.0160348415374756,
      "rewards/rejected": -2.9825170040130615,
      "step": 1030
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1769689822475147e-07,
      "logits/chosen": -1.5879560708999634,
      "logits/rejected": -1.5036201477050781,
      "logps/chosen": -1936.926025390625,
      "logps/rejected": -2708.7578125,
      "loss": 0.4947,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7738593816757202,
      "rewards/margins": 0.7560992240905762,
      "rewards/rejected": -2.529958486557007,
      "step": 1040
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.798797596089351e-07,
      "logits/chosen": -1.4568753242492676,
      "logits/rejected": -1.477905511856079,
      "logps/chosen": -2428.654296875,
      "logps/rejected": -3266.05712890625,
      "loss": 0.5201,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.2049851417541504,
      "rewards/margins": 0.8197803497314453,
      "rewards/rejected": -3.0247654914855957,
      "step": 1050
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4371582698185636e-07,
      "logits/chosen": -1.6677932739257812,
      "logits/rejected": -1.5246306657791138,
      "logps/chosen": -2556.70751953125,
      "logps/rejected": -3194.56884765625,
      "loss": 0.5391,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.3062329292297363,
      "rewards/margins": 0.6159250140190125,
      "rewards/rejected": -2.9221577644348145,
      "step": 1060
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.092332998903416e-07,
      "logits/chosen": -1.7090551853179932,
      "logits/rejected": -1.4194791316986084,
      "logps/chosen": -2150.77392578125,
      "logps/rejected": -2783.882568359375,
      "loss": 0.5195,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9338016510009766,
      "rewards/margins": 0.6355168223381042,
      "rewards/rejected": -2.5693182945251465,
      "step": 1070
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.764590667717562e-07,
      "logits/chosen": -1.606188178062439,
      "logits/rejected": -1.4986306428909302,
      "logps/chosen": -2124.823974609375,
      "logps/rejected": -2895.18310546875,
      "loss": 0.4628,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.912660837173462,
      "rewards/margins": 0.7604308724403381,
      "rewards/rejected": -2.673091411590576,
      "step": 1080
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.454186839872158e-07,
      "logits/chosen": -1.5546796321868896,
      "logits/rejected": -1.550550103187561,
      "logps/chosen": -2096.05078125,
      "logps/rejected": -2596.4248046875,
      "loss": 0.5839,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.9206135272979736,
      "rewards/margins": 0.4739322066307068,
      "rewards/rejected": -2.394545793533325,
      "step": 1090
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1613635589349756e-07,
      "logits/chosen": -1.657184362411499,
      "logits/rejected": -1.3406727313995361,
      "logps/chosen": -2227.37841796875,
      "logps/rejected": -3146.83349609375,
      "loss": 0.459,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9884710311889648,
      "rewards/margins": 0.9205359220504761,
      "rewards/rejected": -2.9090065956115723,
      "step": 1100
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8863491596921745e-07,
      "logits/chosen": -1.4567536115646362,
      "logits/rejected": -1.1964176893234253,
      "logps/chosen": -2488.02099609375,
      "logps/rejected": -3093.317626953125,
      "loss": 0.5213,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.2580440044403076,
      "rewards/margins": 0.6173666715621948,
      "rewards/rejected": -2.875410556793213,
      "step": 1110
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.629358090099639e-07,
      "logits/chosen": -1.8676036596298218,
      "logits/rejected": -1.5324196815490723,
      "logps/chosen": -2476.497314453125,
      "logps/rejected": -3140.97607421875,
      "loss": 0.5292,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.2105040550231934,
      "rewards/margins": 0.6873103380203247,
      "rewards/rejected": -2.8978142738342285,
      "step": 1120
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.3905907440629752e-07,
      "logits/chosen": -1.5062824487686157,
      "logits/rejected": -1.7191336154937744,
      "logps/chosen": -2101.674560546875,
      "logps/rejected": -2914.34228515625,
      "loss": 0.4889,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8925731182098389,
      "rewards/margins": 0.7410818338394165,
      "rewards/rejected": -2.633654832839966,
      "step": 1130
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1702333051763271e-07,
      "logits/chosen": -1.885053277015686,
      "logits/rejected": -1.7132413387298584,
      "logps/chosen": -2271.51806640625,
      "logps/rejected": -3019.444580078125,
      "loss": 0.4746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0059704780578613,
      "rewards/margins": 0.8035289645195007,
      "rewards/rejected": -2.809499740600586,
      "step": 1140
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.684576015420277e-08,
      "logits/chosen": -1.7674312591552734,
      "logits/rejected": -1.7866573333740234,
      "logps/chosen": -2304.083740234375,
      "logps/rejected": -3042.02783203125,
      "loss": 0.5235,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.0314338207244873,
      "rewards/margins": 0.7121332883834839,
      "rewards/rejected": -2.7435669898986816,
      "step": 1150
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.854209717842231e-08,
      "logits/chosen": -1.6722157001495361,
      "logits/rejected": -1.738372564315796,
      "logps/chosen": -2533.139404296875,
      "logps/rejected": -3169.639892578125,
      "loss": 0.508,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.2765398025512695,
      "rewards/margins": 0.6401499509811401,
      "rewards/rejected": -2.91668963432312,
      "step": 1160
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.212661423609184e-08,
      "logits/chosen": -1.6138194799423218,
      "logits/rejected": -1.4168260097503662,
      "logps/chosen": -2651.40087890625,
      "logps/rejected": -2953.78564453125,
      "loss": 0.5389,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.369499683380127,
      "rewards/margins": 0.3613986372947693,
      "rewards/rejected": -2.730898380279541,
      "step": 1170
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.761211162702117e-08,
      "logits/chosen": -1.8268349170684814,
      "logits/rejected": -1.389704942703247,
      "logps/chosen": -2308.206298828125,
      "logps/rejected": -3084.591796875,
      "loss": 0.4561,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0564706325531006,
      "rewards/margins": 0.8305153846740723,
      "rewards/rejected": -2.8869857788085938,
      "step": 1180
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.5009907323737826e-08,
      "logits/chosen": -1.5044516324996948,
      "logits/rejected": -1.3144376277923584,
      "logps/chosen": -2185.82275390625,
      "logps/rejected": -3079.09130859375,
      "loss": 0.4552,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9693256616592407,
      "rewards/margins": 0.8803497552871704,
      "rewards/rejected": -2.8496756553649902,
      "step": 1190
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4329828146074096e-08,
      "logits/chosen": -1.5807933807373047,
      "logits/rejected": -1.4424628019332886,
      "logps/chosen": -2456.582763671875,
      "logps/rejected": -2916.781982421875,
      "loss": 0.5377,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -2.223078489303589,
      "rewards/margins": 0.4909304976463318,
      "rewards/rejected": -2.7140090465545654,
      "step": 1200
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.5580202098509078e-08,
      "logits/chosen": -1.4222395420074463,
      "logits/rejected": -1.4579776525497437,
      "logps/chosen": -2549.04052734375,
      "logps/rejected": -3383.83203125,
      "loss": 0.4934,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.3147170543670654,
      "rewards/margins": 0.7796028852462769,
      "rewards/rejected": -3.094320058822632,
      "step": 1210
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.767851876239075e-09,
      "logits/chosen": -1.9085347652435303,
      "logits/rejected": -1.5333627462387085,
      "logps/chosen": -2555.34228515625,
      "logps/rejected": -3125.24853515625,
      "loss": 0.5644,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.2714717388153076,
      "rewards/margins": 0.6149500012397766,
      "rewards/rejected": -2.8864216804504395,
      "step": 1220
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.8980895450474455e-09,
      "logits/chosen": -1.6516891717910767,
      "logits/rejected": -1.3506808280944824,
      "logps/chosen": -2481.84228515625,
      "logps/rejected": -3454.458984375,
      "loss": 0.4893,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.2125697135925293,
      "rewards/margins": 1.0183699131011963,
      "rewards/rejected": -3.2309398651123047,
      "step": 1230
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.747123991141193e-10,
      "logits/chosen": -1.2762700319290161,
      "logits/rejected": -1.1082122325897217,
      "logps/chosen": -2387.552490234375,
      "logps/rejected": -3240.255615234375,
      "loss": 0.4634,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.163398265838623,
      "rewards/margins": 0.8387178182601929,
      "rewards/rejected": -3.0021164417266846,
      "step": 1240
    },
    {
      "epoch": 1.0,
      "learning_rate": 0.0,
      "logits/chosen": -1.7946316003799438,
      "logits/rejected": -1.654803991317749,
      "logps/chosen": -2349.287109375,
      "logps/rejected": -2943.7216796875,
      "loss": 0.547,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.1279029846191406,
      "rewards/margins": 0.613613486289978,
      "rewards/rejected": -2.74151611328125,
      "step": 1250
    },
    {
      "epoch": 1.0,
      "step": 1250,
      "total_flos": 0.0,
      "train_loss": 0.5607085178375244,
      "train_runtime": 13233.1545,
      "train_samples_per_second": 1.134,
      "train_steps_per_second": 0.094
    }
  ],
  "logging_steps": 10,
  "max_steps": 1250,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 20,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}