diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -25,1761 +25,1761 @@
     {
       "epoch": 0.01,
       "learning_rate": 4.0000000000000003e-07,
-      "logits/chosen": -2.477782726287842,
-      "logits/rejected": -2.1347055435180664,
-      "logps/chosen": -285.4302978515625,
-      "logps/rejected": -191.6398468017578,
+      "logits/chosen": -2.477527141571045,
+      "logits/rejected": -2.134815216064453,
+      "logps/chosen": -285.37506103515625,
+      "logps/rejected": -191.59552001953125,
       "loss": 0.6932,
-      "rewards/accuracies": 0.3611111044883728,
-      "rewards/chosen": -7.325204205699265e-05,
-      "rewards/margins": -3.685199044412002e-05,
-      "rewards/rejected": -3.6400058888830245e-05,
+      "rewards/accuracies": 0.2777777910232544,
+      "rewards/chosen": -1.798523953766562e-05,
+      "rewards/margins": -2.5926061425707303e-05,
+      "rewards/rejected": 7.940820069052279e-06,
       "step": 10
     },
     {
       "epoch": 0.02,
       "learning_rate": 8.000000000000001e-07,
-      "logits/chosen": -2.361138343811035,
-      "logits/rejected": -2.1276116371154785,
-      "logps/chosen": -271.57037353515625,
-      "logps/rejected": -208.7891387939453,
-      "loss": 0.6932,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -4.8331461584893987e-05,
-      "rewards/margins": -0.00016093224985525012,
-      "rewards/rejected": 0.00011260081373620778,
+      "logits/chosen": -2.360628843307495,
+      "logits/rejected": -2.1267056465148926,
+      "logps/chosen": -271.4191589355469,
+      "logps/rejected": -208.81991577148438,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.00010290103818988428,
+      "rewards/margins": 2.105976818711497e-05,
+      "rewards/rejected": 8.184127364074811e-05,
       "step": 20
     },
     {
       "epoch": 0.02,
       "learning_rate": 1.2000000000000002e-06,
-      "logits/chosen": -2.2788968086242676,
-      "logits/rejected": -2.2496988773345947,
-      "logps/chosen": -269.85345458984375,
-      "logps/rejected": -288.97467041015625,
+      "logits/chosen": -2.278747797012329,
+      "logits/rejected": -2.2493417263031006,
+      "logps/chosen": -269.8002624511719,
+      "logps/rejected": -288.9651794433594,
       "loss": 0.6931,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.00023421745572704822,
-      "rewards/margins": 0.00014686177019029856,
-      "rewards/rejected": 8.735566370887682e-05,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.00028741464484483004,
+      "rewards/margins": 0.00019061131752096117,
+      "rewards/rejected": 9.68033418757841e-05,
       "step": 30
     },
     {
       "epoch": 0.03,
       "learning_rate": 1.6000000000000001e-06,
-      "logits/chosen": -2.508540153503418,
-      "logits/rejected": -2.340709924697876,
-      "logps/chosen": -210.4788055419922,
-      "logps/rejected": -181.59371948242188,
+      "logits/chosen": -2.5087287425994873,
+      "logits/rejected": -2.340841293334961,
+      "logps/chosen": -210.5767059326172,
+      "logps/rejected": -181.60897827148438,
       "loss": 0.6931,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.0002574770478531718,
-      "rewards/margins": 0.000190830382052809,
-      "rewards/rejected": 6.664668035227805e-05,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": 0.0001595711219124496,
+      "rewards/margins": 0.00010818429291248322,
+      "rewards/rejected": 5.138682899996638e-05,
       "step": 40
     },
     {
       "epoch": 0.04,
       "learning_rate": 2.0000000000000003e-06,
-      "logits/chosen": -2.3627312183380127,
-      "logits/rejected": -2.338744640350342,
-      "logps/chosen": -195.59091186523438,
-      "logps/rejected": -211.3479461669922,
+      "logits/chosen": -2.362971067428589,
+      "logits/rejected": -2.338986873626709,
+      "logps/chosen": -195.54049682617188,
+      "logps/rejected": -211.3101806640625,
       "loss": 0.6931,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.0001339252630714327,
-      "rewards/margins": 0.00029316064319573343,
-      "rewards/rejected": -0.00015923538012430072,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0001843376230681315,
+      "rewards/margins": 0.00030581915052607656,
+      "rewards/rejected": -0.0001214815056300722,
       "step": 50
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.4000000000000003e-06,
-      "logits/chosen": -2.495098829269409,
-      "logits/rejected": -2.304222583770752,
-      "logps/chosen": -244.5299530029297,
-      "logps/rejected": -277.59429931640625,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.0008799650822766125,
-      "rewards/margins": -0.0001361109607387334,
-      "rewards/rejected": -0.0007438541506417096,
+      "logits/chosen": -2.495109796524048,
+      "logits/rejected": -2.304320812225342,
+      "logps/chosen": -244.63357543945312,
+      "logps/rejected": -277.849853515625,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.000983591889962554,
+      "rewards/margins": 1.5824800357222557e-05,
+      "rewards/rejected": -0.0009994168067350984,
       "step": 60
     },
     {
       "epoch": 0.06,
       "learning_rate": 2.8000000000000003e-06,
-      "logits/chosen": -2.249950885772705,
-      "logits/rejected": -2.266622543334961,
-      "logps/chosen": -229.8421173095703,
-      "logps/rejected": -216.9417266845703,
+      "logits/chosen": -2.2505688667297363,
+      "logits/rejected": -2.2676665782928467,
+      "logps/chosen": -229.9136199951172,
+      "logps/rejected": -216.98001098632812,
       "loss": 0.6928,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.001011955551803112,
-      "rewards/margins": 0.0009981539333239198,
-      "rewards/rejected": -0.00201010936871171,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0010834664572030306,
+      "rewards/margins": 0.0009649285930208862,
+      "rewards/rejected": -0.0020483951084315777,
       "step": 70
     },
     {
       "epoch": 0.06,
       "learning_rate": 3.2000000000000003e-06,
-      "logits/chosen": -2.4004368782043457,
-      "logits/rejected": -2.394763708114624,
-      "logps/chosen": -265.89312744140625,
-      "logps/rejected": -262.71856689453125,
+      "logits/chosen": -2.4005239009857178,
+      "logits/rejected": -2.3944191932678223,
+      "logps/chosen": -266.11859130859375,
+      "logps/rejected": -262.9701232910156,
       "loss": 0.6925,
       "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.0012450494104996324,
-      "rewards/margins": 0.0020160204730927944,
-      "rewards/rejected": -0.0032610702328383923,
+      "rewards/chosen": -0.0014705440262332559,
+      "rewards/margins": 0.002042059786617756,
+      "rewards/rejected": -0.00351260369643569,
       "step": 80
     },
     {
       "epoch": 0.07,
       "learning_rate": 3.6000000000000003e-06,
-      "logits/chosen": -2.3297479152679443,
-      "logits/rejected": -1.9925298690795898,
-      "logps/chosen": -268.21148681640625,
-      "logps/rejected": -179.89309692382812,
+      "logits/chosen": -2.3290226459503174,
+      "logits/rejected": -1.9915828704833984,
+      "logps/chosen": -268.29278564453125,
+      "logps/rejected": -180.13323974609375,
       "loss": 0.6924,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.002822208683937788,
-      "rewards/margins": 0.002088978886604309,
-      "rewards/rejected": -0.004911187570542097,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.002903540385887027,
+      "rewards/margins": 0.002247781725600362,
+      "rewards/rejected": -0.005151322111487389,
       "step": 90
     },
     {
       "epoch": 0.08,
       "learning_rate": 4.000000000000001e-06,
-      "logits/chosen": -2.3094894886016846,
-      "logits/rejected": -2.3130946159362793,
-      "logps/chosen": -275.2729187011719,
-      "logps/rejected": -251.2887420654297,
+      "logits/chosen": -2.3091585636138916,
+      "logits/rejected": -2.312863826751709,
+      "logps/chosen": -275.3695373535156,
+      "logps/rejected": -251.3533477783203,
       "loss": 0.6928,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.003516852855682373,
-      "rewards/margins": 0.0011107095051556826,
-      "rewards/rejected": -0.004627562593668699,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.003613454522565007,
+      "rewards/margins": 0.0010787051869556308,
+      "rewards/rejected": -0.004692160524427891,
       "step": 100
     },
     {
       "epoch": 0.09,
       "learning_rate": 4.4e-06,
-      "logits/chosen": -2.2586567401885986,
-      "logits/rejected": -2.1448559761047363,
-      "logps/chosen": -211.8444366455078,
-      "logps/rejected": -180.26292419433594,
+      "logits/chosen": -2.258510112762451,
+      "logits/rejected": -2.1448404788970947,
+      "logps/chosen": -211.90158081054688,
+      "logps/rejected": -180.40475463867188,
       "loss": 0.6919,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.00023485408746637404,
-      "rewards/margins": 0.00365343876183033,
-      "rewards/rejected": -0.003418584819883108,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0001777430734364316,
+      "rewards/margins": 0.0037381600122898817,
+      "rewards/rejected": -0.003560416866093874,
       "step": 110
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.800000000000001e-06,
-      "logits/chosen": -2.1576130390167236,
-      "logits/rejected": -2.168747663497925,
-      "logps/chosen": -238.6951141357422,
-      "logps/rejected": -270.9164123535156,
-      "loss": 0.6905,
+      "logits/chosen": -2.158743381500244,
+      "logits/rejected": -2.169588088989258,
+      "logps/chosen": -238.22067260742188,
+      "logps/rejected": -270.46417236328125,
+      "loss": 0.6906,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.0023627025075256824,
-      "rewards/margins": 0.005031881853938103,
-      "rewards/rejected": -0.0073945834301412106,
+      "rewards/chosen": -0.0018882494186982512,
+      "rewards/margins": 0.005054115317761898,
+      "rewards/rejected": -0.006942364387214184,
       "step": 120
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999756310023261e-06,
-      "logits/chosen": -2.439880132675171,
-      "logits/rejected": -2.3125338554382324,
-      "logps/chosen": -240.32211303710938,
-      "logps/rejected": -199.84616088867188,
-      "loss": 0.691,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.006134279537945986,
-      "rewards/margins": 0.007197326980531216,
-      "rewards/rejected": -0.013331606984138489,
+      "logits/chosen": -2.4432594776153564,
+      "logits/rejected": -2.315918445587158,
+      "logps/chosen": -237.79501342773438,
+      "logps/rejected": -196.10853576660156,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0036071953363716602,
+      "rewards/margins": 0.005986797157675028,
+      "rewards/rejected": -0.009593991562724113,
       "step": 130
     },
     {
       "epoch": 0.11,
       "learning_rate": 4.997807075247147e-06,
-      "logits/chosen": -2.3178114891052246,
-      "logits/rejected": -2.017031192779541,
-      "logps/chosen": -214.5780792236328,
-      "logps/rejected": -194.8052215576172,
+      "logits/chosen": -2.3266425132751465,
+      "logits/rejected": -2.025289535522461,
+      "logps/chosen": -207.1509552001953,
+      "logps/rejected": -187.79754638671875,
       "loss": 0.6907,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.023361103609204292,
-      "rewards/margins": 0.004234091378748417,
-      "rewards/rejected": -0.027595192193984985,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.015933997929096222,
+      "rewards/margins": 0.004653518553823233,
+      "rewards/rejected": -0.02058752067387104,
       "step": 140
     },
     {
       "epoch": 0.12,
       "learning_rate": 4.993910125649561e-06,
-      "logits/chosen": -2.2956302165985107,
-      "logits/rejected": -2.1826136112213135,
-      "logps/chosen": -245.3561248779297,
-      "logps/rejected": -219.53140258789062,
-      "loss": 0.687,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.01726583205163479,
-      "rewards/margins": 0.01438895333558321,
-      "rewards/rejected": -0.031654782593250275,
+      "logits/chosen": -2.303725481033325,
+      "logits/rejected": -2.190458297729492,
+      "logps/chosen": -239.7921600341797,
+      "logps/rejected": -213.1902618408203,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.01170186698436737,
+      "rewards/margins": 0.013611750677227974,
+      "rewards/rejected": -0.025313619524240494,
       "step": 150
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.988068499954578e-06,
-      "logits/chosen": -2.1682140827178955,
-      "logits/rejected": -2.240255117416382,
-      "logps/chosen": -334.9776306152344,
-      "logps/rejected": -354.7172546386719,
-      "loss": 0.6901,
+      "logits/chosen": -2.1805403232574463,
+      "logits/rejected": -2.2514309883117676,
+      "logps/chosen": -323.44732666015625,
+      "logps/rejected": -342.3636779785156,
+      "loss": 0.6899,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.08232413232326508,
-      "rewards/margins": 0.013598999008536339,
-      "rewards/rejected": -0.09592311829328537,
+      "rewards/chosen": -0.07079382240772247,
+      "rewards/margins": 0.012775696814060211,
+      "rewards/rejected": -0.08356951922178268,
       "step": 160
     },
     {
       "epoch": 0.14,
       "learning_rate": 4.980286753286196e-06,
-      "logits/chosen": -2.3772664070129395,
-      "logits/rejected": -1.992201566696167,
-      "logps/chosen": -375.8896789550781,
-      "logps/rejected": -332.3999938964844,
+      "logits/chosen": -2.377260446548462,
+      "logits/rejected": -1.992498755455017,
+      "logps/chosen": -361.740966796875,
+      "logps/rejected": -316.5578308105469,
       "loss": 0.6815,
       "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.10898187011480331,
-      "rewards/margins": -0.0016051859129220247,
-      "rewards/rejected": -0.10737669467926025,
+      "rewards/chosen": -0.09483315795660019,
+      "rewards/margins": -0.003298636060208082,
+      "rewards/rejected": -0.09153451770544052,
       "step": 170
     },
     {
       "epoch": 0.14,
       "learning_rate": 4.970570953616383e-06,
-      "logits/chosen": -2.130321979522705,
-      "logits/rejected": -2.113255739212036,
-      "logps/chosen": -351.16204833984375,
-      "logps/rejected": -367.5383605957031,
+      "logits/chosen": -2.125253677368164,
+      "logits/rejected": -2.1080162525177,
+      "logps/chosen": -345.4566955566406,
+      "logps/rejected": -362.8345642089844,
       "loss": 0.6796,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.12460009753704071,
-      "rewards/margins": 0.016373703256249428,
-      "rewards/rejected": -0.1409737914800644,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.11889471858739853,
+      "rewards/margins": 0.017375323921442032,
+      "rewards/rejected": -0.13627007603645325,
       "step": 180
     },
     {
       "epoch": 0.15,
       "learning_rate": 4.958928677033465e-06,
-      "logits/chosen": -1.8712676763534546,
-      "logits/rejected": -1.8069908618927002,
-      "logps/chosen": -588.4874877929688,
-      "logps/rejected": -609.7493286132812,
-      "loss": 0.6845,
+      "logits/chosen": -1.872900366783142,
+      "logits/rejected": -1.8023853302001953,
+      "logps/chosen": -607.0579223632812,
+      "logps/rejected": -631.1737060546875,
+      "loss": 0.6814,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.3521875739097595,
-      "rewards/margins": 0.048989322036504745,
-      "rewards/rejected": -0.40117692947387695,
+      "rewards/chosen": -0.37075790762901306,
+      "rewards/margins": 0.051843322813510895,
+      "rewards/rejected": -0.42260122299194336,
       "step": 190
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.9453690018345144e-06,
-      "logits/chosen": -2.2647531032562256,
-      "logits/rejected": -2.1226019859313965,
-      "logps/chosen": -324.08404541015625,
-      "logps/rejected": -342.01953125,
-      "loss": 0.6819,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.0697219967842102,
-      "rewards/margins": 0.010161248967051506,
-      "rewards/rejected": -0.07988324016332626,
+      "logits/chosen": -2.2707138061523438,
+      "logits/rejected": -2.1255440711975098,
+      "logps/chosen": -350.2916564941406,
+      "logps/rejected": -372.66778564453125,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.09592956304550171,
+      "rewards/margins": 0.014601891860365868,
+      "rewards/rejected": -0.11053146421909332,
       "step": 200
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.9299025014463665e-06,
-      "logits/chosen": -2.41994047164917,
-      "logits/rejected": -2.061509132385254,
-      "logps/chosen": -352.439453125,
-      "logps/rejected": -327.6614685058594,
-      "loss": 0.6733,
+      "logits/chosen": -2.4437155723571777,
+      "logits/rejected": -2.0841479301452637,
+      "logps/chosen": -348.2846984863281,
+      "logps/rejected": -332.79693603515625,
+      "loss": 0.6716,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.0789322555065155,
-      "rewards/margins": 0.058253444731235504,
-      "rewards/rejected": -0.1371857076883316,
+      "rewards/chosen": -0.07477747648954391,
+      "rewards/margins": 0.06754375249147415,
+      "rewards/rejected": -0.14232121407985687,
       "step": 210
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.912541236180779e-06,
-      "logits/chosen": -2.14202618598938,
-      "logits/rejected": -1.7346794605255127,
-      "logps/chosen": -420.63775634765625,
-      "logps/rejected": -467.1897888183594,
-      "loss": 0.6749,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.1840709000825882,
-      "rewards/margins": 0.0854046419262886,
-      "rewards/rejected": -0.2694755494594574,
+      "logits/chosen": -2.2131965160369873,
+      "logits/rejected": -1.79428231716156,
+      "logps/chosen": -453.1966247558594,
+      "logps/rejected": -504.7650451660156,
+      "loss": 0.6753,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.2166297435760498,
+      "rewards/margins": 0.09042102098464966,
+      "rewards/rejected": -0.30705076456069946,
       "step": 220
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.893298743830168e-06,
-      "logits/chosen": -2.003080129623413,
-      "logits/rejected": -2.0397982597351074,
-      "logps/chosen": -470.3719787597656,
-      "logps/rejected": -551.4140625,
-      "loss": 0.676,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.27153176069259644,
-      "rewards/margins": 0.054350595921278,
-      "rewards/rejected": -0.3258823752403259,
+      "logits/chosen": -2.114473342895508,
+      "logits/rejected": -2.1522789001464844,
+      "logps/chosen": -447.54510498046875,
+      "logps/rejected": -531.5616455078125,
+      "loss": 0.6747,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.24870488047599792,
+      "rewards/margins": 0.05732503533363342,
+      "rewards/rejected": -0.30602994561195374,
       "step": 230
     },
     {
       "epoch": 0.19,
       "learning_rate": 4.8721900291112415e-06,
-      "logits/chosen": -1.852705717086792,
-      "logits/rejected": -1.6845359802246094,
-      "logps/chosen": -642.0089111328125,
-      "logps/rejected": -671.2232055664062,
-      "loss": 0.6746,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.4353899359703064,
-      "rewards/margins": 0.041978295892477036,
-      "rewards/rejected": -0.4773682653903961,
+      "logits/chosen": -2.0410103797912598,
+      "logits/rejected": -1.8702392578125,
+      "logps/chosen": -417.52557373046875,
+      "logps/rejected": -418.7784118652344,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.21090665459632874,
+      "rewards/margins": 0.014016765169799328,
+      "rewards/rejected": -0.22492341697216034,
       "step": 240
     },
     {
       "epoch": 0.2,
       "learning_rate": 4.849231551964771e-06,
-      "logits/chosen": -1.8547617197036743,
-      "logits/rejected": -1.7584753036499023,
-      "logps/chosen": -806.89208984375,
-      "logps/rejected": -791.40869140625,
-      "loss": 0.6877,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.6369258165359497,
-      "rewards/margins": -0.01818418875336647,
-      "rewards/rejected": -0.6187416315078735,
+      "logits/chosen": -1.9370673894882202,
+      "logits/rejected": -1.845969796180725,
+      "logps/chosen": -467.31817626953125,
+      "logps/rejected": -516.6582641601562,
+      "loss": 0.6586,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.2973518967628479,
+      "rewards/margins": 0.04663931205868721,
+      "rewards/rejected": -0.34399116039276123,
       "step": 250
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.824441214720629e-06,
-      "logits/chosen": -1.795576810836792,
-      "logits/rejected": -1.5512839555740356,
-      "logps/chosen": -1120.3328857421875,
-      "logps/rejected": -1224.4669189453125,
-      "loss": 0.6485,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.8469240069389343,
-      "rewards/margins": 0.12044433504343033,
-      "rewards/rejected": -0.9673683047294617,
+      "logits/chosen": -1.7340936660766602,
+      "logits/rejected": -1.4695367813110352,
+      "logps/chosen": -1242.895751953125,
+      "logps/rejected": -1449.67138671875,
+      "loss": 0.6468,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.9694870114326477,
+      "rewards/margins": 0.2230859100818634,
+      "rewards/rejected": -1.192572832107544,
       "step": 260
     },
     {
       "epoch": 0.22,
       "learning_rate": 4.7978383481380865e-06,
-      "logits/chosen": -1.3707473278045654,
-      "logits/rejected": -1.1816232204437256,
-      "logps/chosen": -1378.314697265625,
-      "logps/rejected": -1494.1234130859375,
-      "loss": 0.6689,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -1.1299686431884766,
-      "rewards/margins": 0.15295067429542542,
-      "rewards/rejected": -1.2829192876815796,
+      "logits/chosen": -1.9577367305755615,
+      "logits/rejected": -1.7384374141693115,
+      "logps/chosen": -651.7098999023438,
+      "logps/rejected": -757.5595703125,
+      "loss": 0.6567,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.40336376428604126,
+      "rewards/margins": 0.14299169182777405,
+      "rewards/rejected": -0.5463554263114929,
       "step": 270
     },
     {
       "epoch": 0.22,
       "learning_rate": 4.769443696332272e-06,
-      "logits/chosen": -1.5147939920425415,
-      "logits/rejected": -1.5684055089950562,
-      "logps/chosen": -509.37579345703125,
-      "logps/rejected": -719.7078247070312,
-      "loss": 0.6556,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.3430081605911255,
-      "rewards/margins": 0.17507921159267426,
-      "rewards/rejected": -0.5180873274803162,
+      "logits/chosen": -1.6682405471801758,
+      "logits/rejected": -1.7225072383880615,
+      "logps/chosen": -636.3197021484375,
+      "logps/rejected": -823.1611328125,
+      "loss": 0.654,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.4699520468711853,
+      "rewards/margins": 0.15158866345882416,
+      "rewards/rejected": -0.6215407252311707,
       "step": 280
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7392794005985324e-06,
-      "logits/chosen": -1.9780447483062744,
-      "logits/rejected": -1.7068169116973877,
-      "logps/chosen": -503.57684326171875,
-      "logps/rejected": -538.3973388671875,
-      "loss": 0.6529,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.2268252670764923,
-      "rewards/margins": 0.09029386937618256,
-      "rewards/rejected": -0.31711912155151367,
+      "logits/chosen": -1.7123403549194336,
+      "logits/rejected": -1.4536840915679932,
+      "logps/chosen": -984.7986450195312,
+      "logps/rejected": -1116.86328125,
+      "loss": 0.6296,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.7080470323562622,
+      "rewards/margins": 0.18753795325756073,
+      "rewards/rejected": -0.8955849409103394,
       "step": 290
     },
     {
       "epoch": 0.24,
       "learning_rate": 4.707368982147318e-06,
-      "logits/chosen": -1.725359320640564,
-      "logits/rejected": -1.6483027935028076,
-      "logps/chosen": -612.9721069335938,
-      "logps/rejected": -744.4105224609375,
-      "loss": 0.6381,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.32202452421188354,
-      "rewards/margins": 0.21170559525489807,
-      "rewards/rejected": -0.533730149269104,
+      "logits/chosen": -1.5288642644882202,
+      "logits/rejected": -1.4273738861083984,
+      "logps/chosen": -1024.952880859375,
+      "logps/rejected": -1212.50439453125,
+      "loss": 0.6232,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.734005331993103,
+      "rewards/margins": 0.26781877875328064,
+      "rewards/rejected": -1.0018240213394165,
       "step": 300
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.673737323763048e-06,
-      "logits/chosen": -1.478655457496643,
-      "logits/rejected": -1.5181188583374023,
-      "logps/chosen": -718.6035766601562,
-      "logps/rejected": -861.0172119140625,
-      "loss": 0.6412,
+      "logits/chosen": -1.244638204574585,
+      "logits/rejected": -1.2806625366210938,
+      "logps/chosen": -906.6008911132812,
+      "logps/rejected": -1112.8660888671875,
+      "loss": 0.6264,
       "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.5259624719619751,
-      "rewards/margins": 0.10101442039012909,
-      "rewards/rejected": -0.6269768476486206,
+      "rewards/chosen": -0.713959813117981,
+      "rewards/margins": 0.16486592590808868,
+      "rewards/rejected": -0.8788257837295532,
       "step": 310
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.638410650401267e-06,
-      "logits/chosen": -1.3320460319519043,
-      "logits/rejected": -1.0306655168533325,
-      "logps/chosen": -665.1239013671875,
-      "logps/rejected": -827.1819458007812,
-      "loss": 0.6595,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.4110494554042816,
-      "rewards/margins": 0.24086549878120422,
-      "rewards/rejected": -0.6519149541854858,
+      "logits/chosen": -1.2582708597183228,
+      "logits/rejected": -0.9609068632125854,
+      "logps/chosen": -789.2811889648438,
+      "logps/rejected": -924.0436401367188,
+      "loss": 0.6751,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.53520667552948,
+      "rewards/margins": 0.21356996893882751,
+      "rewards/rejected": -0.7487767338752747,
       "step": 320
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.601416508739211e-06,
-      "logits/chosen": -1.3477035760879517,
-      "logits/rejected": -1.3697832822799683,
-      "logps/chosen": -867.5467529296875,
-      "logps/rejected": -948.0695190429688,
-      "loss": 0.621,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.5947723388671875,
-      "rewards/margins": 0.1193898469209671,
-      "rewards/rejected": -0.7141622304916382,
+      "logits/chosen": -1.5856783390045166,
+      "logits/rejected": -1.6001255512237549,
+      "logps/chosen": -586.2677001953125,
+      "logps/rejected": -678.0260009765625,
+      "loss": 0.6333,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.31349319219589233,
+      "rewards/margins": 0.13062533736228943,
+      "rewards/rejected": -0.44411858916282654,
       "step": 330
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.562783745695738e-06,
-      "logits/chosen": -1.3015021085739136,
-      "logits/rejected": -1.025702714920044,
-      "logps/chosen": -1093.9085693359375,
-      "logps/rejected": -1216.8255615234375,
-      "loss": 0.6451,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.9000682830810547,
-      "rewards/margins": 0.1583687961101532,
-      "rewards/rejected": -1.0584371089935303,
+      "logits/chosen": -1.710599660873413,
+      "logits/rejected": -1.4391021728515625,
+      "logps/chosen": -415.0328674316406,
+      "logps/rejected": -529.9607543945312,
+      "loss": 0.6341,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.2211925983428955,
+      "rewards/margins": 0.15037958323955536,
+      "rewards/rejected": -0.3715721666812897,
       "step": 340
     },
     {
       "epoch": 0.28,
       "learning_rate": 4.522542485937369e-06,
-      "logits/chosen": -1.5709688663482666,
-      "logits/rejected": -1.283427119255066,
-      "logps/chosen": -1376.850341796875,
-      "logps/rejected": -1516.5667724609375,
-      "loss": 0.6395,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -1.1210005283355713,
-      "rewards/margins": 0.1694861352443695,
-      "rewards/rejected": -1.2904865741729736,
+      "logits/chosen": -1.8084720373153687,
+      "logits/rejected": -1.506037712097168,
+      "logps/chosen": -570.231201171875,
+      "logps/rejected": -703.8088989257812,
+      "loss": 0.6278,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.31438136100769043,
+      "rewards/margins": 0.16334742307662964,
+      "rewards/rejected": -0.4777289032936096,
       "step": 350
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4807241083879774e-06,
-      "logits/chosen": -1.6043704748153687,
-      "logits/rejected": -1.3832875490188599,
-      "logps/chosen": -1154.26220703125,
-      "logps/rejected": -1336.6424560546875,
-      "loss": 0.6427,
+      "logits/chosen": -1.4908154010772705,
+      "logits/rejected": -1.2486298084259033,
+      "logps/chosen": -1008.43310546875,
+      "logps/rejected": -1258.6966552734375,
+      "loss": 0.6064,
       "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.896734356880188,
-      "rewards/margins": 0.21884091198444366,
-      "rewards/rejected": -1.1155753135681152,
+      "rewards/chosen": -0.7509051561355591,
+      "rewards/margins": 0.28672417998313904,
+      "rewards/rejected": -1.0376293659210205,
       "step": 360
     },
     {
       "epoch": 0.3,
       "learning_rate": 4.437361221760449e-06,
-      "logits/chosen": -1.3710479736328125,
-      "logits/rejected": -1.3098589181900024,
-      "logps/chosen": -1009.2317504882812,
-      "logps/rejected": -1062.556640625,
-      "loss": 0.6655,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.821437656879425,
-      "rewards/margins": 0.057194460183382034,
-      "rewards/rejected": -0.8786321878433228,
+      "logits/chosen": -1.114203929901123,
+      "logits/rejected": -1.0544617176055908,
+      "logps/chosen": -935.9404296875,
+      "logps/rejected": -1014.9347534179688,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.7481463551521301,
+      "rewards/margins": 0.08286388218402863,
+      "rewards/rejected": -0.8310102224349976,
       "step": 370
     },
     {
       "epoch": 0.3,
       "learning_rate": 4.3924876391293915e-06,
-      "logits/chosen": -0.8956707119941711,
-      "logits/rejected": -0.8277013897895813,
-      "logps/chosen": -1106.165283203125,
-      "logps/rejected": -1343.876708984375,
-      "loss": 0.6192,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.9171901941299438,
-      "rewards/margins": 0.2286480963230133,
-      "rewards/rejected": -1.1458381414413452,
+      "logits/chosen": -0.82923823595047,
+      "logits/rejected": -0.7714365124702454,
+      "logps/chosen": -1037.5648193359375,
+      "logps/rejected": -1317.840087890625,
+      "loss": 0.6065,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.8485895991325378,
+      "rewards/margins": 0.2712119519710541,
+      "rewards/rejected": -1.1198015213012695,
       "step": 380
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.346138351564711e-06,
-      "logits/chosen": -0.6417841911315918,
-      "logits/rejected": -0.44266366958618164,
-      "logps/chosen": -836.3943481445312,
-      "logps/rejected": -1194.9322509765625,
-      "loss": 0.6001,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.644795298576355,
-      "rewards/margins": 0.4003651738166809,
-      "rewards/rejected": -1.0451605319976807,
+      "logits/chosen": -0.4438857138156891,
+      "logits/rejected": -0.24459032714366913,
+      "logps/chosen": -1160.9703369140625,
+      "logps/rejected": -1448.4993896484375,
+      "loss": 0.6151,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.9693711996078491,
+      "rewards/margins": 0.3293563425540924,
+      "rewards/rejected": -1.2987276315689087,
       "step": 390
     },
     {
       "epoch": 0.32,
       "learning_rate": 4.2983495008466285e-06,
-      "logits/chosen": -0.8112918734550476,
-      "logits/rejected": -0.7515584230422974,
-      "logps/chosen": -834.2249755859375,
-      "logps/rejected": -1054.40087890625,
-      "loss": 0.6716,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.6205351948738098,
-      "rewards/margins": 0.20678479969501495,
-      "rewards/rejected": -0.8273199796676636,
+      "logits/chosen": -1.0683261156082153,
+      "logits/rejected": -1.0331823825836182,
+      "logps/chosen": -835.9192504882812,
+      "logps/rejected": -1117.007568359375,
+      "loss": 0.672,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.6222294569015503,
+      "rewards/margins": 0.26769739389419556,
+      "rewards/rejected": -0.8899267911911011,
       "step": 400
     },
     {
       "epoch": 0.33,
       "learning_rate": 4.249158351283414e-06,
-      "logits/chosen": -1.344972848892212,
-      "logits/rejected": -1.1616101264953613,
-      "logps/chosen": -608.2757568359375,
-      "logps/rejected": -736.3763427734375,
-      "loss": 0.6591,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.3484669029712677,
-      "rewards/margins": 0.12206283956766129,
-      "rewards/rejected": -0.4705297350883484,
+      "logits/chosen": -1.524524450302124,
+      "logits/rejected": -1.3127562999725342,
+      "logps/chosen": -700.7545166015625,
+      "logps/rejected": -866.1266479492188,
+      "loss": 0.6623,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4409456253051758,
+      "rewards/margins": 0.15933458507061005,
+      "rewards/rejected": -0.600280225276947,
       "step": 410
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.198603260653792e-06,
-      "logits/chosen": -1.5138790607452393,
-      "logits/rejected": -1.3168789148330688,
-      "logps/chosen": -660.6969604492188,
-      "logps/rejected": -721.4447631835938,
-      "loss": 0.6268,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.39333969354629517,
-      "rewards/margins": 0.0856555625796318,
-      "rewards/rejected": -0.4789952337741852,
+      "logits/chosen": -1.8277499675750732,
+      "logits/rejected": -1.6153600215911865,
+      "logps/chosen": -648.5438842773438,
+      "logps/rejected": -690.0060424804688,
+      "loss": 0.6354,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.3811865448951721,
+      "rewards/margins": 0.06636995077133179,
+      "rewards/rejected": -0.4475564956665039,
       "step": 420
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.146723650296701e-06,
-      "logits/chosen": -0.835279643535614,
-      "logits/rejected": -0.7461940050125122,
-      "logps/chosen": -1367.2841796875,
-      "logps/rejected": -1730.0777587890625,
-      "loss": 0.6363,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.0674763917922974,
-      "rewards/margins": 0.35012298822402954,
-      "rewards/rejected": -1.4175993204116821,
+      "logits/chosen": -1.4686410427093506,
+      "logits/rejected": -1.4586213827133179,
+      "logps/chosen": -1219.008544921875,
+      "logps/rejected": -1602.6322021484375,
+      "loss": 0.6145,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9192007184028625,
+      "rewards/margins": 0.37095293402671814,
+      "rewards/rejected": -1.2901536226272583,
       "step": 430
     },
     {
       "epoch": 0.35,
       "learning_rate": 4.093559974371725e-06,
-      "logits/chosen": 0.03398115560412407,
-      "logits/rejected": 0.2700420320034027,
-      "logps/chosen": -1314.656005859375,
-      "logps/rejected": -1531.143798828125,
-      "loss": 0.5735,
+      "logits/chosen": -1.156723976135254,
+      "logits/rejected": -0.8702109456062317,
+      "logps/chosen": -1515.7261962890625,
+      "logps/rejected": -1630.477783203125,
+      "loss": 0.5933,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -1.1307179927825928,
-      "rewards/margins": 0.2540770173072815,
-      "rewards/rejected": -1.3847953081130981,
+      "rewards/chosen": -1.331788420677185,
+      "rewards/margins": 0.15234079957008362,
+      "rewards/rejected": -1.4841291904449463,
       "step": 440
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.039153688314146e-06,
-      "logits/chosen": -1.2114994525909424,
-      "logits/rejected": -0.9774553179740906,
-      "logps/chosen": -639.7308959960938,
-      "logps/rejected": -1037.600830078125,
-      "loss": 0.6237,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.37664055824279785,
-      "rewards/margins": 0.4325637221336365,
-      "rewards/rejected": -0.8092042803764343,
+      "logits/chosen": -1.4883257150650024,
+      "logits/rejected": -1.2767736911773682,
+      "logps/chosen": -1196.55517578125,
+      "logps/rejected": -1548.430419921875,
+      "loss": 0.6235,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.9334648847579956,
+      "rewards/margins": 0.3865690231323242,
+      "rewards/rejected": -1.3200337886810303,
       "step": 450
     },
     {
       "epoch": 0.37,
       "learning_rate": 3.983547216509254e-06,
-      "logits/chosen": -1.3054085969924927,
-      "logits/rejected": -1.221360683441162,
-      "logps/chosen": -556.9669799804688,
-      "logps/rejected": -634.5089111328125,
-      "loss": 0.6412,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.32842499017715454,
-      "rewards/margins": 0.10787343978881836,
-      "rewards/rejected": -0.4362984299659729,
+      "logits/chosen": -1.688001275062561,
+      "logits/rejected": -1.5989247560501099,
+      "logps/chosen": -713.7869873046875,
+      "logps/rejected": -806.5700073242188,
+      "loss": 0.6501,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.4852449297904968,
+      "rewards/margins": 0.12311458587646484,
+      "rewards/rejected": -0.6083595752716064,
       "step": 460
     },
     {
       "epoch": 0.38,
       "learning_rate": 3.92678391921108e-06,
-      "logits/chosen": -1.2041904926300049,
-      "logits/rejected": -1.1369637250900269,
-      "logps/chosen": -508.9098205566406,
-      "logps/rejected": -549.3627319335938,
-      "loss": 0.6646,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.34369945526123047,
-      "rewards/margins": 0.057983528822660446,
-      "rewards/rejected": -0.4016830027103424,
+      "logits/chosen": -1.6326487064361572,
+      "logits/rejected": -1.5402270555496216,
+      "logps/chosen": -553.8263549804688,
+      "logps/rejected": -636.650390625,
+      "loss": 0.6405,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.3886161148548126,
+      "rewards/margins": 0.10035456717014313,
+      "rewards/rejected": -0.48897066712379456,
       "step": 470
     },
     {
       "epoch": 0.38,
       "learning_rate": 3.868908058731376e-06,
-      "logits/chosen": -1.0662200450897217,
-      "logits/rejected": -0.960114598274231,
-      "logps/chosen": -994.2325439453125,
-      "logps/rejected": -1141.496337890625,
-      "loss": 0.6428,
+      "logits/chosen": -1.2314743995666504,
+      "logits/rejected": -1.1016968488693237,
+      "logps/chosen": -1220.119140625,
+      "logps/rejected": -1464.5045166015625,
+      "loss": 0.627,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.8284689784049988,
-      "rewards/margins": 0.12653744220733643,
-      "rewards/rejected": -0.95500648021698,
+      "rewards/chosen": -1.0543556213378906,
+      "rewards/margins": 0.22365888953208923,
+      "rewards/rejected": -1.2780145406723022,
       "step": 480
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8099647649251984e-06,
-      "logits/chosen": -1.110414743423462,
-      "logits/rejected": -0.8326665759086609,
-      "logps/chosen": -1098.611572265625,
-      "logps/rejected": -1337.651123046875,
-      "loss": 0.5994,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.8522143363952637,
-      "rewards/margins": 0.2896762788295746,
-      "rewards/rejected": -1.141890525817871,
+      "logits/chosen": -1.1137750148773193,
+      "logits/rejected": -0.8115717768669128,
+      "logps/chosen": -1465.813720703125,
+      "logps/rejected": -1871.6728515625,
+      "loss": 0.5908,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2194162607192993,
+      "rewards/margins": 0.45649608969688416,
+      "rewards/rejected": -1.6759124994277954,
       "step": 490
     },
     {
       "epoch": 0.4,
       "learning_rate": 3.7500000000000005e-06,
-      "logits/chosen": -1.0082452297210693,
-      "logits/rejected": -0.8550570607185364,
-      "logps/chosen": -1118.4150390625,
-      "logps/rejected": -1403.4140625,
-      "loss": 0.6138,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.9241348505020142,
-      "rewards/margins": 0.283856064081192,
-      "rewards/rejected": -1.2079908847808838,
+      "logits/chosen": -1.160712480545044,
+      "logits/rejected": -1.0044102668762207,
+      "logps/chosen": -1403.710693359375,
+      "logps/rejected": -1854.1767578125,
+      "loss": 0.6275,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2094306945800781,
+      "rewards/margins": 0.44932323694229126,
+      "rewards/rejected": -1.658753752708435,
       "step": 500
     },
     {
       "epoch": 0.41,
       "learning_rate": 3.689060522675689e-06,
-      "logits/chosen": -1.1805860996246338,
-      "logits/rejected": -1.0648720264434814,
-      "logps/chosen": -968.7009887695312,
-      "logps/rejected": -1127.3431396484375,
-      "loss": 0.5993,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.7270181775093079,
-      "rewards/margins": 0.19340118765830994,
-      "rewards/rejected": -0.9204193949699402,
+      "logits/chosen": -1.4455738067626953,
+      "logits/rejected": -1.3064179420471191,
+      "logps/chosen": -1048.9417724609375,
+      "logps/rejected": -1300.092529296875,
+      "loss": 0.5972,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.8072590827941895,
+      "rewards/margins": 0.2859098017215729,
+      "rewards/rejected": -1.0931689739227295,
       "step": 510
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.627193851723577e-06,
-      "logits/chosen": -1.0413739681243896,
-      "logits/rejected": -0.9251521825790405,
-      "logps/chosen": -962.42919921875,
-      "logps/rejected": -1037.397705078125,
-      "loss": 0.6336,
+      "logits/chosen": -1.168351650238037,
+      "logits/rejected": -1.0151941776275635,
+      "logps/chosen": -1202.1624755859375,
+      "logps/rejected": -1375.6861572265625,
+      "loss": 0.6332,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.7336166501045227,
-      "rewards/margins": 0.10032469034194946,
-      "rewards/rejected": -0.8339413404464722,
+      "rewards/chosen": -0.973349928855896,
+      "rewards/margins": 0.1988798826932907,
+      "rewards/rejected": -1.1722297668457031,
       "step": 520
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.564448228912682e-06,
-      "logits/chosen": -1.196645975112915,
-      "logits/rejected": -1.1510522365570068,
-      "logps/chosen": -906.9932861328125,
-      "logps/rejected": -1005.1427612304688,
-      "loss": 0.6233,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.6604412794113159,
-      "rewards/margins": 0.08722977340221405,
-      "rewards/rejected": -0.7476710081100464,
+      "logits/chosen": -1.2055370807647705,
+      "logits/rejected": -1.1897690296173096,
+      "logps/chosen": -1302.2347412109375,
+      "logps/rejected": -1396.9376220703125,
+      "loss": 0.6383,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.0556827783584595,
+      "rewards/margins": 0.08378318697214127,
+      "rewards/rejected": -1.1394660472869873,
       "step": 530
     },
     {
       "epoch": 0.43,
       "learning_rate": 3.5008725813922383e-06,
-      "logits/chosen": -1.0590436458587646,
-      "logits/rejected": -0.7820866703987122,
-      "logps/chosen": -1043.649169921875,
-      "logps/rejected": -1336.0374755859375,
-      "loss": 0.5875,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.7807156443595886,
-      "rewards/margins": 0.3648682236671448,
-      "rewards/rejected": -1.1455838680267334,
+      "logits/chosen": -1.2467204332351685,
+      "logits/rejected": -0.9966660737991333,
+      "logps/chosen": -1237.6484375,
+      "logps/rejected": -1357.805908203125,
+      "loss": 0.6061,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.9747149348258972,
+      "rewards/margins": 0.1926373690366745,
+      "rewards/rejected": -1.167352318763733,
       "step": 540
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.436516483539781e-06,
-      "logits/chosen": -0.8871183395385742,
-      "logits/rejected": -0.6445604562759399,
-      "logps/chosen": -974.1388549804688,
-      "logps/rejected": -1435.357666015625,
-      "loss": 0.5548,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.7723373770713806,
-      "rewards/margins": 0.4823000431060791,
-      "rewards/rejected": -1.2546374797821045,
+      "logits/chosen": -1.0729809999465942,
+      "logits/rejected": -0.880784809589386,
+      "logps/chosen": -1176.840576171875,
+      "logps/rejected": -1703.155029296875,
+      "loss": 0.5811,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.9750388860702515,
+      "rewards/margins": 0.547395646572113,
+      "rewards/rejected": -1.5224344730377197,
       "step": 550
     },
     {
       "epoch": 0.45,
       "learning_rate": 3.3714301183045382e-06,
-      "logits/chosen": -1.1284804344177246,
-      "logits/rejected": -1.0746089220046997,
-      "logps/chosen": -1098.0726318359375,
-      "logps/rejected": -1571.875732421875,
-      "loss": 0.5809,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.8001961708068848,
-      "rewards/margins": 0.49779874086380005,
-      "rewards/rejected": -1.29799485206604,
+      "logits/chosen": -1.343149185180664,
+      "logits/rejected": -1.3310397863388062,
+      "logps/chosen": -1393.66748046875,
+      "logps/rejected": -1949.219482421875,
+      "loss": 0.593,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.0957911014556885,
+      "rewards/margins": 0.5795475244522095,
+      "rewards/rejected": -1.6753385066986084,
       "step": 560
     },
     {
       "epoch": 0.46,
       "learning_rate": 3.3056642380762783e-06,
-      "logits/chosen": -1.0926157236099243,
-      "logits/rejected": -0.9620027542114258,
-      "logps/chosen": -909.43896484375,
-      "logps/rejected": -1261.0572509765625,
-      "loss": 0.5876,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.6548485159873962,
-      "rewards/margins": 0.3792751133441925,
-      "rewards/rejected": -1.0341236591339111,
+      "logits/chosen": -1.2191109657287598,
+      "logits/rejected": -1.1489986181259155,
+      "logps/chosen": -1165.780517578125,
+      "logps/rejected": -1509.78173828125,
+      "loss": 0.5879,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.9111903309822083,
+      "rewards/margins": 0.37165799736976624,
+      "rewards/rejected": -1.2828481197357178,
       "step": 570
     },
     {
       "epoch": 0.46,
       "learning_rate": 3.2392701251101172e-06,
-      "logits/chosen": -1.1053316593170166,
-      "logits/rejected": -0.9159544706344604,
-      "logps/chosen": -904.6321411132812,
-      "logps/rejected": -1194.8170166015625,
-      "loss": 0.5921,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.6870709657669067,
-      "rewards/margins": 0.3379691243171692,
-      "rewards/rejected": -1.0250400304794312,
+      "logits/chosen": -1.2665627002716064,
+      "logits/rejected": -1.0979241132736206,
+      "logps/chosen": -1186.7427978515625,
+      "logps/rejected": -1492.552001953125,
+      "loss": 0.601,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.9691814184188843,
+      "rewards/margins": 0.35359352827072144,
+      "rewards/rejected": -1.3227750062942505,
       "step": 580
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.1722995515381644e-06,
-      "logits/chosen": -0.5783677101135254,
-      "logits/rejected": -0.381605327129364,
-      "logps/chosen": -1375.947265625,
-      "logps/rejected": -1693.8013916015625,
-      "loss": 0.5704,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.1249635219573975,
-      "rewards/margins": 0.3614503741264343,
-      "rewards/rejected": -1.486413836479187,
+      "logits/chosen": -1.089277982711792,
+      "logits/rejected": -0.9281560778617859,
+      "logps/chosen": -1328.541015625,
+      "logps/rejected": -1635.415771484375,
+      "loss": 0.5736,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.0775573253631592,
+      "rewards/margins": 0.35047078132629395,
+      "rewards/rejected": -1.4280281066894531,
       "step": 590
     },
     {
       "epoch": 0.48,
       "learning_rate": 3.1048047389991693e-06,
-      "logits/chosen": -0.6121073365211487,
-      "logits/rejected": -0.4031047224998474,
-      "logps/chosen": -1350.5516357421875,
-      "logps/rejected": -1793.6240234375,
-      "loss": 0.6244,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -1.1342486143112183,
-      "rewards/margins": 0.43501371145248413,
-      "rewards/rejected": -1.5692622661590576,
+      "logits/chosen": -1.0954294204711914,
+      "logits/rejected": -0.9223726391792297,
+      "logps/chosen": -1286.0498046875,
+      "logps/rejected": -1733.7152099609375,
+      "loss": 0.6187,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.0697466135025024,
+      "rewards/margins": 0.43960708379745483,
+      "rewards/rejected": -1.5093533992767334,
       "step": 600
     },
     {
       "epoch": 0.49,
       "learning_rate": 3.0368383179176584e-06,
-      "logits/chosen": -0.8454495668411255,
-      "logits/rejected": -0.6569667458534241,
-      "logps/chosen": -890.9841918945312,
-      "logps/rejected": -1315.620849609375,
-      "loss": 0.5818,
+      "logits/chosen": -1.2414100170135498,
+      "logits/rejected": -1.0788267850875854,
+      "logps/chosen": -808.4361572265625,
+      "logps/rejected": -1251.864990234375,
+      "loss": 0.5898,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.6493796110153198,
-      "rewards/margins": 0.46156787872314453,
-      "rewards/rejected": -1.1109474897384644,
+      "rewards/chosen": -0.566831648349762,
+      "rewards/margins": 0.4803600311279297,
+      "rewards/rejected": -1.0471916198730469,
       "step": 610
     },
     {
       "epoch": 0.5,
       "learning_rate": 2.9684532864643123e-06,
-      "logits/chosen": -0.8009635210037231,
-      "logits/rejected": -0.6670125126838684,
-      "logps/chosen": -1004.8250122070312,
-      "logps/rejected": -1155.979736328125,
-      "loss": 0.6091,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.7524474263191223,
-      "rewards/margins": 0.18189628422260284,
-      "rewards/rejected": -0.934343695640564,
+      "logits/chosen": -1.262020230293274,
+      "logits/rejected": -1.148590087890625,
+      "logps/chosen": -931.0750122070312,
+      "logps/rejected": -1080.2626953125,
+      "loss": 0.6143,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.6786974668502808,
+      "rewards/margins": 0.17992933094501495,
+      "rewards/rejected": -0.8586267232894897,
       "step": 620
     },
     {
       "epoch": 0.5,
       "learning_rate": 2.8997029692295875e-06,
-      "logits/chosen": -0.9459114074707031,
-      "logits/rejected": -0.6500396728515625,
-      "logps/chosen": -1066.767333984375,
-      "logps/rejected": -1516.01953125,
-      "loss": 0.5912,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.7964303493499756,
-      "rewards/margins": 0.4783499240875244,
-      "rewards/rejected": -1.274780511856079,
+      "logits/chosen": -1.380244255065918,
+      "logits/rejected": -1.1846643686294556,
+      "logps/chosen": -1102.616943359375,
+      "logps/rejected": -1461.800048828125,
+      "loss": 0.6248,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.832280158996582,
+      "rewards/margins": 0.3882806599140167,
+      "rewards/rejected": -1.2205607891082764,
       "step": 630
     },
     {
       "epoch": 0.51,
       "learning_rate": 2.8306409756428067e-06,
-      "logits/chosen": -0.7831882238388062,
-      "logits/rejected": -0.3093162477016449,
-      "logps/chosen": -1089.9415283203125,
-      "logps/rejected": -1740.2158203125,
-      "loss": 0.5638,
+      "logits/chosen": -1.1531554460525513,
+      "logits/rejected": -0.73602694272995,
+      "logps/chosen": -1035.087158203125,
+      "logps/rejected": -1619.76171875,
+      "loss": 0.568,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.7907674908638,
-      "rewards/margins": 0.7186497449874878,
-      "rewards/rejected": -1.509417176246643,
+      "rewards/chosen": -0.7359131574630737,
+      "rewards/margins": 0.6530500650405884,
+      "rewards/rejected": -1.388963222503662,
       "step": 640
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.761321158169134e-06,
-      "logits/chosen": -0.5406681299209595,
-      "logits/rejected": -0.27648812532424927,
-      "logps/chosen": -1686.183349609375,
-      "logps/rejected": -1993.3765869140625,
-      "loss": 0.5719,
+      "logits/chosen": -1.1167099475860596,
+      "logits/rejected": -0.873482346534729,
+      "logps/chosen": -1361.4727783203125,
+      "logps/rejected": -1630.313232421875,
+      "loss": 0.5607,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.4113540649414062,
-      "rewards/margins": 0.37404680252075195,
-      "rewards/rejected": -1.7854007482528687,
+      "rewards/chosen": -1.0866434574127197,
+      "rewards/margins": 0.33569416403770447,
+      "rewards/rejected": -1.4223374128341675,
       "step": 650
     },
     {
       "epoch": 0.53,
       "learning_rate": 2.6917975703170466e-06,
-      "logits/chosen": -0.5704952478408813,
-      "logits/rejected": -0.023378795012831688,
-      "logps/chosen": -1294.9827880859375,
-      "logps/rejected": -2076.658447265625,
-      "loss": 0.5743,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.0449953079223633,
-      "rewards/margins": 0.7736978530883789,
-      "rewards/rejected": -1.8186931610107422,
+      "logits/chosen": -0.7387585639953613,
+      "logits/rejected": -0.3248792886734009,
+      "logps/chosen": -1398.619384765625,
+      "logps/rejected": -2202.85498046875,
+      "loss": 0.5724,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1486316919326782,
+      "rewards/margins": 0.7962583303451538,
+      "rewards/rejected": -1.944890022277832,
       "step": 660
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.6221244244890336e-06,
-      "logits/chosen": -0.5983051061630249,
-      "logits/rejected": -0.35873037576675415,
-      "logps/chosen": -1279.24072265625,
-      "logps/rejected": -1712.7236328125,
-      "loss": 0.5588,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.0362498760223389,
-      "rewards/margins": 0.4681357741355896,
-      "rewards/rejected": -1.5043857097625732,
+      "logits/chosen": -0.6219555139541626,
+      "logits/rejected": -0.4327964782714844,
+      "logps/chosen": -1998.990234375,
+      "logps/rejected": -2370.507080078125,
+      "loss": 0.576,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.755999207496643,
+      "rewards/margins": 0.406169593334198,
+      "rewards/rejected": -2.1621687412261963,
       "step": 670
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.5523560497083927e-06,
-      "logits/chosen": -0.7439953088760376,
-      "logits/rejected": -0.5419555306434631,
-      "logps/chosen": -986.34765625,
-      "logps/rejected": -1177.267333984375,
-      "loss": 0.6399,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.7895917296409607,
-      "rewards/margins": 0.20669332146644592,
-      "rewards/rejected": -0.996285080909729,
+      "logits/chosen": -0.9830737113952637,
+      "logits/rejected": -0.8169624209403992,
+      "logps/chosen": -1028.17578125,
+      "logps/rejected": -1266.951904296875,
+      "loss": 0.6217,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.8314197659492493,
+      "rewards/margins": 0.2545499801635742,
+      "rewards/rejected": -1.0859696865081787,
       "step": 680
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.482546849255096e-06,
-      "logits/chosen": -0.9641045331954956,
-      "logits/rejected": -0.6751854419708252,
-      "logps/chosen": -834.6181640625,
-      "logps/rejected": -1149.5087890625,
-      "loss": 0.5592,
+      "logits/chosen": -1.0446968078613281,
+      "logits/rejected": -0.7680097818374634,
+      "logps/chosen": -963.1312255859375,
+      "logps/rejected": -1276.6685791015625,
+      "loss": 0.5595,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.6201135516166687,
-      "rewards/margins": 0.3461000919342041,
-      "rewards/rejected": -0.966213583946228,
+      "rewards/chosen": -0.7486265301704407,
+      "rewards/margins": 0.34474682807922363,
+      "rewards/rejected": -1.0933732986450195,
       "step": 690
     },
     {
       "epoch": 0.56,
       "learning_rate": 2.4127512582437486e-06,
-      "logits/chosen": -0.9225761294364929,
-      "logits/rejected": -0.772851824760437,
-      "logps/chosen": -925.7664184570312,
-      "logps/rejected": -1246.7874755859375,
-      "loss": 0.5751,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.6425189971923828,
-      "rewards/margins": 0.3423493504524231,
-      "rewards/rejected": -0.9848684072494507,
+      "logits/chosen": -0.9387643933296204,
+      "logits/rejected": -0.8350385427474976,
+      "logps/chosen": -1224.355712890625,
+      "logps/rejected": -1547.6513671875,
+      "loss": 0.5682,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9411084055900574,
+      "rewards/margins": 0.34462398290634155,
+      "rewards/rejected": -1.285732388496399,
       "step": 700
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3430237011767166e-06,
-      "logits/chosen": -0.45442017912864685,
-      "logits/rejected": -0.23890094459056854,
-      "logps/chosen": -1335.2960205078125,
-      "logps/rejected": -1824.8148193359375,
-      "loss": 0.5562,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -1.1126158237457275,
-      "rewards/margins": 0.5235506296157837,
-      "rewards/rejected": -1.6361665725708008,
+      "logits/chosen": -0.7177497148513794,
+      "logits/rejected": -0.5354570746421814,
+      "logps/chosen": -1501.9898681640625,
+      "logps/rejected": -2006.5120849609375,
+      "loss": 0.5666,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.279309630393982,
+      "rewards/margins": 0.5385541319847107,
+      "rewards/rejected": -1.8178638219833374,
       "step": 710
     },
     {
       "epoch": 0.58,
       "learning_rate": 2.2734185495055503e-06,
-      "logits/chosen": 0.43471193313598633,
-      "logits/rejected": 0.7335575222969055,
-      "logps/chosen": -1717.737060546875,
-      "logps/rejected": -2255.147705078125,
-      "loss": 0.5663,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.4929530620574951,
-      "rewards/margins": 0.5923892259597778,
-      "rewards/rejected": -2.0853421688079834,
+      "logits/chosen": -0.14772020280361176,
+      "logits/rejected": 0.09259579330682755,
+      "logps/chosen": -1891.1800537109375,
+      "logps/rejected": -2354.27734375,
+      "loss": 0.5845,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.666395902633667,
+      "rewards/margins": 0.5180760025978088,
+      "rewards/rejected": -2.184471845626831,
       "step": 720
     },
     {
       "epoch": 0.58,
       "learning_rate": 2.2039900792337477e-06,
-      "logits/chosen": -0.5602989792823792,
-      "logits/rejected": -0.28851318359375,
-      "logps/chosen": -1234.8255615234375,
-      "logps/rejected": -1552.613037109375,
-      "loss": 0.5947,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.978289783000946,
-      "rewards/margins": 0.3596613109111786,
-      "rewards/rejected": -1.3379510641098022,
+      "logits/chosen": -0.7175928354263306,
+      "logits/rejected": -0.47789135575294495,
+      "logps/chosen": -1339.34765625,
+      "logps/rejected": -1721.939453125,
+      "loss": 0.6008,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.0828118324279785,
+      "rewards/margins": 0.42446571588516235,
+      "rewards/rejected": -1.5072776079177856,
       "step": 730
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.134792428593971e-06,
-      "logits/chosen": -0.9215381741523743,
-      "logits/rejected": -0.6034310460090637,
-      "logps/chosen": -918.3829345703125,
-      "logps/rejected": -1260.914794921875,
-      "loss": 0.5731,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.6723597645759583,
-      "rewards/margins": 0.3877628743648529,
-      "rewards/rejected": -1.0601227283477783,
+      "logits/chosen": -1.1001973152160645,
+      "logits/rejected": -0.7717048525810242,
+      "logps/chosen": -975.1065673828125,
+      "logps/rejected": -1379.069580078125,
+      "loss": 0.5754,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7290834188461304,
+      "rewards/margins": 0.4491938054561615,
+      "rewards/rejected": -1.1782772541046143,
       "step": 740
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0658795558326745e-06,
-      "logits/chosen": -0.75888991355896,
-      "logits/rejected": -0.8145910501480103,
-      "logps/chosen": -942.7371215820312,
-      "logps/rejected": -1151.885986328125,
-      "loss": 0.6138,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.7426967620849609,
-      "rewards/margins": 0.20792725682258606,
-      "rewards/rejected": -0.9506238698959351,
+      "logits/chosen": -0.9087737202644348,
+      "logits/rejected": -0.9751186370849609,
+      "logps/chosen": -962.6619873046875,
+      "logps/rejected": -1202.0980224609375,
+      "loss": 0.6048,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.7626216411590576,
+      "rewards/margins": 0.23821432888507843,
+      "rewards/rejected": -1.0008360147476196,
       "step": 750
     },
     {
       "epoch": 0.61,
       "learning_rate": 1.997305197135089e-06,
-      "logits/chosen": -0.5050958395004272,
-      "logits/rejected": -0.3710903823375702,
-      "logps/chosen": -1115.988525390625,
-      "logps/rejected": -1241.006103515625,
-      "loss": 0.6371,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.9049081802368164,
-      "rewards/margins": 0.14062346518039703,
-      "rewards/rejected": -1.0455316305160522,
+      "logits/chosen": -0.6515249013900757,
+      "logits/rejected": -0.5174766778945923,
+      "logps/chosen": -1231.8673095703125,
+      "logps/rejected": -1379.020263671875,
+      "loss": 0.6427,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.020787000656128,
+      "rewards/margins": 0.1627589762210846,
+      "rewards/rejected": -1.1835458278656006,
       "step": 760
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.9291228247233607e-06,
-      "logits/chosen": -1.0052917003631592,
-      "logits/rejected": -0.7385827302932739,
-      "logps/chosen": -1185.632568359375,
-      "logps/rejected": -1218.97412109375,
-      "loss": 0.602,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.8952628970146179,
-      "rewards/margins": 0.1188296228647232,
-      "rewards/rejected": -1.0140926837921143,
+      "logits/chosen": -1.1023533344268799,
+      "logits/rejected": -0.8313691020011902,
+      "logps/chosen": -1354.640380859375,
+      "logps/rejected": -1333.6419677734375,
+      "loss": 0.6151,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.0642707347869873,
+      "rewards/margins": 0.0644897073507309,
+      "rewards/rejected": -1.1287604570388794,
       "step": 770
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8613856051605242e-06,
-      "logits/chosen": -0.7601789236068726,
-      "logits/rejected": -0.4364476203918457,
-      "logps/chosen": -1060.472412109375,
-      "logps/rejected": -1608.5179443359375,
-      "loss": 0.5912,
+      "logits/chosen": -0.8817610740661621,
+      "logits/rejected": -0.6308177709579468,
+      "logps/chosen": -1051.3311767578125,
+      "logps/rejected": -1573.681396484375,
+      "loss": 0.6084,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.8007382154464722,
-      "rewards/margins": 0.5700600743293762,
-      "rewards/rejected": -1.3707983493804932,
+      "rewards/chosen": -0.7915970087051392,
+      "rewards/margins": 0.5443645715713501,
+      "rewards/rejected": -1.3359615802764893,
       "step": 780
     },
     {
       "epoch": 0.63,
       "learning_rate": 1.7941463578928088e-06,
-      "logits/chosen": -0.8455929756164551,
-      "logits/rejected": -0.7644866704940796,
-      "logps/chosen": -1184.275634765625,
-      "logps/rejected": -1357.3797607421875,
-      "loss": 0.6282,
+      "logits/chosen": -1.0322520732879639,
+      "logits/rejected": -0.9708759188652039,
+      "logps/chosen": -1105.9547119140625,
+      "logps/rejected": -1240.3804931640625,
+      "loss": 0.6281,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.9668749570846558,
-      "rewards/margins": 0.18699315190315247,
-      "rewards/rejected": -1.1538680791854858,
+      "rewards/chosen": -0.8885539174079895,
+      "rewards/margins": 0.14831490814685822,
+      "rewards/rejected": -1.0368688106536865,
       "step": 790
     },
     {
       "epoch": 0.64,
       "learning_rate": 1.7274575140626318e-06,
-      "logits/chosen": -0.6272526979446411,
-      "logits/rejected": -0.6869767904281616,
-      "logps/chosen": -1237.728759765625,
-      "logps/rejected": -1527.4461669921875,
-      "loss": 0.5405,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.9917057156562805,
-      "rewards/margins": 0.2875545024871826,
-      "rewards/rejected": -1.2792601585388184,
+      "logits/chosen": -0.8422597646713257,
+      "logits/rejected": -0.9403928518295288,
+      "logps/chosen": -1141.7333984375,
+      "logps/rejected": -1371.9580078125,
+      "loss": 0.5443,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.8957103490829468,
+      "rewards/margins": 0.22806143760681152,
+      "rewards/rejected": -1.1237719058990479,
       "step": 800
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.661371075624363e-06,
-      "logits/chosen": -0.38267868757247925,
-      "logits/rejected": -0.11700303852558136,
-      "logps/chosen": -1500.436767578125,
-      "logps/rejected": -1794.047119140625,
-      "loss": 0.5961,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -1.308133840560913,
-      "rewards/margins": 0.3339870274066925,
-      "rewards/rejected": -1.6421207189559937,
+      "logits/chosen": -0.6579400300979614,
+      "logits/rejected": -0.38535481691360474,
+      "logps/chosen": -1287.460205078125,
+      "logps/rejected": -1504.084228515625,
+      "loss": 0.6025,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.095157265663147,
+      "rewards/margins": 0.25700071454048157,
+      "rewards/rejected": -1.3521578311920166,
       "step": 810
     },
     {
       "epoch": 0.66,
       "learning_rate": 1.5959385747947697e-06,
-      "logits/chosen": -0.5077565908432007,
-      "logits/rejected": -0.36555030941963196,
-      "logps/chosen": -1464.396240234375,
-      "logps/rejected": -2057.291259765625,
-      "loss": 0.5732,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.2214739322662354,
-      "rewards/margins": 0.5794647336006165,
-      "rewards/rejected": -1.800938606262207,
+      "logits/chosen": -0.7758182287216187,
+      "logits/rejected": -0.67876136302948,
+      "logps/chosen": -1263.397216796875,
+      "logps/rejected": -1797.307373046875,
+      "loss": 0.5785,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.020475149154663,
+      "rewards/margins": 0.5204795002937317,
+      "rewards/rejected": -1.540954828262329,
       "step": 820
     },
     {
       "epoch": 0.66,
       "learning_rate": 1.5312110338697427e-06,
-      "logits/chosen": -0.8566253781318665,
-      "logits/rejected": -0.703198254108429,
-      "logps/chosen": -1135.494873046875,
-      "logps/rejected": -1410.838134765625,
-      "loss": 0.5885,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.9456474184989929,
-      "rewards/margins": 0.2893359661102295,
-      "rewards/rejected": -1.2349833250045776,
+      "logits/chosen": -1.0213903188705444,
+      "logits/rejected": -0.8918190002441406,
+      "logps/chosen": -1065.27734375,
+      "logps/rejected": -1240.601806640625,
+      "loss": 0.6148,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.8754297494888306,
+      "rewards/margins": 0.1893172711133957,
+      "rewards/rejected": -1.0647470951080322,
       "step": 830
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.467238925438646e-06,
-      "logits/chosen": -0.9742065668106079,
-      "logits/rejected": -0.7325109839439392,
-      "logps/chosen": -1128.455078125,
-      "logps/rejected": -1652.849609375,
-      "loss": 0.5826,
+      "logits/chosen": -1.0888749361038208,
+      "logits/rejected": -0.9006432294845581,
+      "logps/chosen": -1159.646728515625,
+      "logps/rejected": -1520.9205322265625,
+      "loss": 0.5909,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.8605993986129761,
-      "rewards/margins": 0.5381206274032593,
-      "rewards/rejected": -1.3987200260162354,
+      "rewards/chosen": -0.891791045665741,
+      "rewards/margins": 0.37499967217445374,
+      "rewards/rejected": -1.2667908668518066,
       "step": 840
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4040721330273063e-06,
-      "logits/chosen": -0.6383851766586304,
-      "logits/rejected": -0.30349114537239075,
-      "logps/chosen": -1556.1243896484375,
-      "logps/rejected": -2115.495361328125,
-      "loss": 0.5752,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.2599270343780518,
-      "rewards/margins": 0.6025987863540649,
-      "rewards/rejected": -1.8625257015228271,
+      "logits/chosen": -0.7614107131958008,
+      "logits/rejected": -0.4498376250267029,
+      "logps/chosen": -1577.4564208984375,
+      "logps/rejected": -2064.27587890625,
+      "loss": 0.5638,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.2812590599060059,
+      "rewards/margins": 0.5300472974777222,
+      "rewards/rejected": -1.811306357383728,
       "step": 850
     },
     {
       "epoch": 0.69,
       "learning_rate": 1.3417599122003464e-06,
-      "logits/chosen": -0.945850670337677,
-      "logits/rejected": -0.4593322277069092,
-      "logps/chosen": -1373.3946533203125,
-      "logps/rejected": -2057.10498046875,
-      "loss": 0.5475,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.1107696294784546,
-      "rewards/margins": 0.7481247782707214,
-      "rewards/rejected": -1.8588945865631104,
+      "logits/chosen": -1.0388845205307007,
+      "logits/rejected": -0.6018816232681274,
+      "logps/chosen": -1428.4647216796875,
+      "logps/rejected": -2024.8896484375,
+      "loss": 0.5664,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1658397912979126,
+      "rewards/margins": 0.6608397364616394,
+      "rewards/rejected": -1.8266795873641968,
       "step": 860
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.280350852153168e-06,
-      "logits/chosen": -0.587804913520813,
-      "logits/rejected": -0.4838123917579651,
-      "logps/chosen": -1164.4200439453125,
-      "logps/rejected": -1821.4710693359375,
-      "loss": 0.522,
+      "logits/chosen": -0.6871947050094604,
+      "logits/rejected": -0.6853007674217224,
+      "logps/chosen": -1243.111572265625,
+      "logps/rejected": -1829.707275390625,
+      "loss": 0.5468,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.9738459587097168,
-      "rewards/margins": 0.613839328289032,
-      "rewards/rejected": -1.5876853466033936,
+      "rewards/chosen": -1.0525375604629517,
+      "rewards/margins": 0.5433839559555054,
+      "rewards/rejected": -1.595921277999878,
       "step": 870
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2198928378235717e-06,
-      "logits/chosen": -0.6612527966499329,
-      "logits/rejected": -0.5115641355514526,
-      "logps/chosen": -1650.718505859375,
-      "logps/rejected": -2091.0732421875,
-      "loss": 0.6349,
+      "logits/chosen": -0.8195822834968567,
+      "logits/rejected": -0.6704593896865845,
+      "logps/chosen": -1643.8544921875,
+      "logps/rejected": -2129.94189453125,
+      "loss": 0.6271,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": -1.4022929668426514,
-      "rewards/margins": 0.423071950674057,
-      "rewards/rejected": -1.8253650665283203,
+      "rewards/chosen": -1.3954288959503174,
+      "rewards/margins": 0.46880459785461426,
+      "rewards/rejected": -1.8642336130142212,
       "step": 880
     },
     {
       "epoch": 0.71,
       "learning_rate": 1.160433012552508e-06,
-      "logits/chosen": -1.0209529399871826,
-      "logits/rejected": -0.7942039966583252,
-      "logps/chosen": -967.7880859375,
-      "logps/rejected": -1502.6470947265625,
-      "loss": 0.5917,
+      "logits/chosen": -1.0698474645614624,
+      "logits/rejected": -0.9074158668518066,
+      "logps/chosen": -1074.9571533203125,
+      "logps/rejected": -1626.334228515625,
+      "loss": 0.5905,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.7300990223884583,
-      "rewards/margins": 0.5862218141555786,
-      "rewards/rejected": -1.316320776939392,
+      "rewards/chosen": -0.8372681736946106,
+      "rewards/margins": 0.602739691734314,
+      "rewards/rejected": -1.4400079250335693,
       "step": 890
     },
     {
       "epoch": 0.72,
       "learning_rate": 1.1020177413231334e-06,
-      "logits/chosen": -0.9528091549873352,
-      "logits/rejected": -0.6705907583236694,
-      "logps/chosen": -1106.0501708984375,
-      "logps/rejected": -1562.62939453125,
-      "loss": 0.5511,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.8729621767997742,
-      "rewards/margins": 0.4861548840999603,
-      "rewards/rejected": -1.359117031097412,
+      "logits/chosen": -1.0306494235992432,
+      "logits/rejected": -0.7876461148262024,
+      "logps/chosen": -1247.2880859375,
+      "logps/rejected": -1743.0517578125,
+      "loss": 0.5594,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.0142000913619995,
+      "rewards/margins": 0.5253391265869141,
+      "rewards/rejected": -1.5395392179489136,
       "step": 900
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0446925746067768e-06,
-      "logits/chosen": -0.6244916915893555,
-      "logits/rejected": -0.30852293968200684,
-      "logps/chosen": -1030.0357666015625,
-      "logps/rejected": -1489.8277587890625,
-      "loss": 0.5651,
+      "logits/chosen": -0.7109737396240234,
+      "logits/rejected": -0.4215407371520996,
+      "logps/chosen": -1115.1761474609375,
+      "logps/rejected": -1571.439697265625,
+      "loss": 0.584,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.8064696192741394,
-      "rewards/margins": 0.5191851258277893,
-      "rewards/rejected": -1.3256547451019287,
+      "rewards/chosen": -0.8916100263595581,
+      "rewards/margins": 0.515656590461731,
+      "rewards/rejected": -1.407266616821289,
       "step": 910
     },
     {
       "epoch": 0.74,
       "learning_rate": 9.88502212844063e-07,
-      "logits/chosen": -0.8371726870536804,
-      "logits/rejected": -0.5608356595039368,
-      "logps/chosen": -1306.61181640625,
-      "logps/rejected": -1983.6080322265625,
-      "loss": 0.6032,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.0330902338027954,
-      "rewards/margins": 0.6798037886619568,
-      "rewards/rejected": -1.712894082069397,
+      "logits/chosen": -0.9104745984077454,
+      "logits/rejected": -0.6836374402046204,
+      "logps/chosen": -1387.066650390625,
+      "logps/rejected": -2122.8583984375,
+      "loss": 0.5959,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1135450601577759,
+      "rewards/margins": 0.7385995388031006,
+      "rewards/rejected": -1.8521445989608765,
       "step": 920
     },
     {
       "epoch": 0.74,
       "learning_rate": 9.334904715888496e-07,
-      "logits/chosen": -0.6907060146331787,
-      "logits/rejected": -0.3094312250614166,
-      "logps/chosen": -1471.054443359375,
-      "logps/rejected": -2111.68701171875,
-      "loss": 0.6023,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.1933256387710571,
-      "rewards/margins": 0.6424576044082642,
-      "rewards/rejected": -1.8357832431793213,
+      "logits/chosen": -0.7981353998184204,
+      "logits/rejected": -0.47408953309059143,
+      "logps/chosen": -1591.907958984375,
+      "logps/rejected": -2268.22802734375,
+      "loss": 0.5996,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.3141790628433228,
+      "rewards/margins": 0.6781451106071472,
+      "rewards/rejected": -1.9923241138458252,
       "step": 930
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.797002473421729e-07,
-      "logits/chosen": -0.6967111825942993,
-      "logits/rejected": -0.7114680409431458,
-      "logps/chosen": -1068.6622314453125,
-      "logps/rejected": -1549.4698486328125,
-      "loss": 0.5634,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.8170648813247681,
-      "rewards/margins": 0.4820033013820648,
-      "rewards/rejected": -1.2990682125091553,
+      "logits/chosen": -0.7773085832595825,
+      "logits/rejected": -0.8014146089553833,
+      "logps/chosen": -1200.51904296875,
+      "logps/rejected": -1684.487060546875,
+      "loss": 0.551,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.948921799659729,
+      "rewards/margins": 0.48516377806663513,
+      "rewards/rejected": -1.434085488319397,
       "step": 940
     },
     {
       "epoch": 0.76,
       "learning_rate": 8.271734841028553e-07,
-      "logits/chosen": -0.6052312850952148,
-      "logits/rejected": -0.38424620032310486,
-      "logps/chosen": -1452.5819091796875,
-      "logps/rejected": -1862.074462890625,
-      "loss": 0.5435,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.1982152462005615,
-      "rewards/margins": 0.44351309537887573,
-      "rewards/rejected": -1.641728162765503,
+      "logits/chosen": -0.7013689279556274,
+      "logits/rejected": -0.48669877648353577,
+      "logps/chosen": -1563.8013916015625,
+      "logps/rejected": -1946.0748291015625,
+      "loss": 0.5625,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3094342947006226,
+      "rewards/margins": 0.41629427671432495,
+      "rewards/rejected": -1.7257286310195923,
       "step": 950
     },
     {
       "epoch": 0.77,
       "learning_rate": 7.759511406608255e-07,
-      "logits/chosen": -0.8322499990463257,
-      "logits/rejected": -0.7050498723983765,
-      "logps/chosen": -1228.91650390625,
-      "logps/rejected": -1504.3065185546875,
-      "loss": 0.5831,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.9939497709274292,
-      "rewards/margins": 0.2843117415904999,
-      "rewards/rejected": -1.2782615423202515,
+      "logits/chosen": -0.8420946002006531,
+      "logits/rejected": -0.7498366832733154,
+      "logps/chosen": -1459.593994140625,
+      "logps/rejected": -1710.6732177734375,
+      "loss": 0.5906,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2246272563934326,
+      "rewards/margins": 0.26000121235847473,
+      "rewards/rejected": -1.484628438949585,
       "step": 960
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.260731586586983e-07,
-      "logits/chosen": -0.6231692433357239,
-      "logits/rejected": -0.5812069773674011,
-      "logps/chosen": -907.5172729492188,
-      "logps/rejected": -1289.426513671875,
-      "loss": 0.5797,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.7803674936294556,
-      "rewards/margins": 0.38126420974731445,
-      "rewards/rejected": -1.1616315841674805,
+      "logits/chosen": -0.6557348370552063,
+      "logits/rejected": -0.6690261363983154,
+      "logps/chosen": -1101.353759765625,
+      "logps/rejected": -1434.849365234375,
+      "loss": 0.5741,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.974203884601593,
+      "rewards/margins": 0.33285054564476013,
+      "rewards/rejected": -1.3070546388626099,
       "step": 970
     },
     {
       "epoch": 0.78,
       "learning_rate": 6.775784314464717e-07,
-      "logits/chosen": -0.9975751638412476,
-      "logits/rejected": -0.5946329236030579,
-      "logps/chosen": -1023.9324340820312,
-      "logps/rejected": -1547.0753173828125,
-      "loss": 0.5251,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.7850741147994995,
-      "rewards/margins": 0.5707739591598511,
-      "rewards/rejected": -1.355847954750061,
+      "logits/chosen": -1.0794492959976196,
+      "logits/rejected": -0.7161605954170227,
+      "logps/chosen": -1141.7310791015625,
+      "logps/rejected": -1709.1611328125,
+      "loss": 0.5292,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.9028726816177368,
+      "rewards/margins": 0.6150614619255066,
+      "rewards/rejected": -1.5179340839385986,
       "step": 980
     },
     {
       "epoch": 0.79,
       "learning_rate": 6.305047737536707e-07,
-      "logits/chosen": -0.577874481678009,
-      "logits/rejected": -0.44077569246292114,
-      "logps/chosen": -1363.0498046875,
-      "logps/rejected": -2003.899658203125,
-      "loss": 0.5644,
+      "logits/chosen": -0.6755369305610657,
+      "logits/rejected": -0.619501531124115,
+      "logps/chosen": -1471.1715087890625,
+      "logps/rejected": -2048.2431640625,
+      "loss": 0.5757,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.1625807285308838,
-      "rewards/margins": 0.6248854398727417,
-      "rewards/rejected": -1.787466287612915,
+      "rewards/chosen": -1.2707023620605469,
+      "rewards/margins": 0.5611073970794678,
+      "rewards/rejected": -1.8318097591400146,
       "step": 990
     },
     {
       "epoch": 0.8,
       "learning_rate": 5.848888922025553e-07,
-      "logits/chosen": -0.8105753064155579,
-      "logits/rejected": -0.919611930847168,
-      "logps/chosen": -1336.093994140625,
-      "logps/rejected": -1582.025146484375,
-      "loss": 0.5633,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -1.0712718963623047,
-      "rewards/margins": 0.27774709463119507,
-      "rewards/rejected": -1.3490190505981445,
+      "logits/chosen": -0.8728944659233093,
+      "logits/rejected": -1.02140212059021,
+      "logps/chosen": -1593.0389404296875,
+      "logps/rejected": -1743.603759765625,
+      "loss": 0.5888,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3282172679901123,
+      "rewards/margins": 0.18238051235675812,
+      "rewards/rejected": -1.510597825050354,
       "step": 1000
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.407663566854008e-07,
-      "logits/chosen": -1.1405055522918701,
-      "logits/rejected": -0.8410784006118774,
-      "logps/chosen": -1108.752197265625,
-      "logps/rejected": -1712.4912109375,
-      "loss": 0.5699,
+      "logits/chosen": -1.2148396968841553,
+      "logits/rejected": -0.9437984228134155,
+      "logps/chosen": -1290.468994140625,
+      "logps/rejected": -1955.718017578125,
+      "loss": 0.5642,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.8489648103713989,
-      "rewards/margins": 0.6527272462844849,
-      "rewards/rejected": -1.5016920566558838,
+      "rewards/chosen": -1.0306814908981323,
+      "rewards/margins": 0.7142370939254761,
+      "rewards/rejected": -1.7449188232421875,
       "step": 1010
     },
     {
       "epoch": 0.82,
       "learning_rate": 4.981715726281666e-07,
-      "logits/chosen": -0.8187993764877319,
-      "logits/rejected": -0.6521504521369934,
-      "logps/chosen": -1444.044189453125,
-      "logps/rejected": -2250.241943359375,
-      "loss": 0.5623,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.1614019870758057,
-      "rewards/margins": 0.798125147819519,
-      "rewards/rejected": -1.9595272541046143,
+      "logits/chosen": -0.9041656255722046,
+      "logits/rejected": -0.7784754633903503,
+      "logps/chosen": -1627.837890625,
+      "logps/rejected": -2467.47314453125,
+      "loss": 0.5837,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.3451955318450928,
+      "rewards/margins": 0.8315626978874207,
+      "rewards/rejected": -2.176758289337158,
       "step": 1020
     },
     {
       "epoch": 0.82,
       "learning_rate": 4.5713775416217884e-07,
-      "logits/chosen": -0.44266295433044434,
-      "logits/rejected": -0.33986443281173706,
-      "logps/chosen": -1182.5404052734375,
-      "logps/rejected": -1922.5198974609375,
-      "loss": 0.5714,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.9850846529006958,
-      "rewards/margins": 0.7335339188575745,
-      "rewards/rejected": -1.718618631362915,
+      "logits/chosen": -0.508884608745575,
+      "logits/rejected": -0.4718368649482727,
+      "logps/chosen": -1368.59765625,
+      "logps/rejected": -2104.90625,
+      "loss": 0.563,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1711418628692627,
+      "rewards/margins": 0.7298630475997925,
+      "rewards/rejected": -1.9010050296783447,
       "step": 1030
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.1769689822475147e-07,
-      "logits/chosen": -0.5250821709632874,
-      "logits/rejected": -0.5564389228820801,
-      "logps/chosen": -1238.247802734375,
-      "logps/rejected": -1787.1693115234375,
-      "loss": 0.5653,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.0797133445739746,
-      "rewards/margins": 0.4955691695213318,
-      "rewards/rejected": -1.5752824544906616,
+      "logits/chosen": -0.5781607031822205,
+      "logits/rejected": -0.6443449258804321,
+      "logps/chosen": -1434.3941650390625,
+      "logps/rejected": -2030.039306640625,
+      "loss": 0.5716,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2758595943450928,
+      "rewards/margins": 0.5422929525375366,
+      "rewards/rejected": -1.818152666091919,
       "step": 1040
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.798797596089351e-07,
-      "logits/chosen": -0.7688647508621216,
-      "logits/rejected": -0.4375014901161194,
-      "logps/chosen": -1243.4844970703125,
-      "logps/rejected": -1739.737548828125,
-      "loss": 0.5426,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.010633111000061,
-      "rewards/margins": 0.519820511341095,
-      "rewards/rejected": -1.5304534435272217,
+      "logits/chosen": -0.8322515487670898,
+      "logits/rejected": -0.5321582555770874,
+      "logps/chosen": -1379.1712646484375,
+      "logps/rejected": -1931.9146728515625,
+      "loss": 0.55,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1463197469711304,
+      "rewards/margins": 0.5763110518455505,
+      "rewards/rejected": -1.7226308584213257,
       "step": 1050
     },
     {
       "epoch": 0.85,
       "learning_rate": 3.4371582698185636e-07,
-      "logits/chosen": -0.8014335632324219,
-      "logits/rejected": -0.5965282917022705,
-      "logps/chosen": -1543.572021484375,
-      "logps/rejected": -2121.01611328125,
-      "loss": 0.5224,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.2631282806396484,
-      "rewards/margins": 0.5626910924911499,
-      "rewards/rejected": -1.8258193731307983,
+      "logits/chosen": -0.9420675039291382,
+      "logits/rejected": -0.7674391269683838,
+      "logps/chosen": -1645.792236328125,
+      "logps/rejected": -2230.914306640625,
+      "loss": 0.5025,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.3653483390808105,
+      "rewards/margins": 0.5703693628311157,
+      "rewards/rejected": -1.9357175827026367,
       "step": 1060
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.092332998903416e-07,
-      "logits/chosen": -0.7670878171920776,
-      "logits/rejected": -0.6641474366188049,
-      "logps/chosen": -1278.5185546875,
-      "logps/rejected": -1890.5849609375,
-      "loss": 0.5691,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.028751015663147,
-      "rewards/margins": 0.6037184000015259,
-      "rewards/rejected": -1.6324694156646729,
+      "logits/chosen": -0.8753671646118164,
+      "logits/rejected": -0.8147989511489868,
+      "logps/chosen": -1408.5784912109375,
+      "logps/rejected": -2042.339599609375,
+      "loss": 0.576,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.158811092376709,
+      "rewards/margins": 0.6254128217697144,
+      "rewards/rejected": -1.7842239141464233,
       "step": 1070
     },
     {
       "epoch": 0.86,
       "learning_rate": 2.764590667717562e-07,
-      "logits/chosen": -0.5815708637237549,
-      "logits/rejected": -0.41486841440200806,
-      "logps/chosen": -1293.8211669921875,
-      "logps/rejected": -1344.2347412109375,
-      "loss": 0.6088,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -1.1181930303573608,
-      "rewards/margins": 0.08898428082466125,
-      "rewards/rejected": -1.2071774005889893,
+      "logits/chosen": -0.6400734186172485,
+      "logits/rejected": -0.46232056617736816,
+      "logps/chosen": -1486.711669921875,
+      "logps/rejected": -1575.9361572265625,
+      "loss": 0.5985,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -1.3110835552215576,
+      "rewards/margins": 0.12779514491558075,
+      "rewards/rejected": -1.4388787746429443,
       "step": 1080
     },
     {
       "epoch": 0.87,
       "learning_rate": 2.454186839872158e-07,
-      "logits/chosen": -0.5545269250869751,
-      "logits/rejected": -0.26859965920448303,
-      "logps/chosen": -1244.134033203125,
-      "logps/rejected": -2037.753173828125,
-      "loss": 0.5342,
+      "logits/chosen": -0.616904616355896,
+      "logits/rejected": -0.3994078040122986,
+      "logps/chosen": -1475.5816650390625,
+      "logps/rejected": -2202.4833984375,
+      "loss": 0.5632,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.0435465574264526,
-      "rewards/margins": 0.803005039691925,
-      "rewards/rejected": -1.846551537513733,
+      "rewards/chosen": -1.2749942541122437,
+      "rewards/margins": 0.7362874746322632,
+      "rewards/rejected": -2.011281728744507,
       "step": 1090
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.1613635589349756e-07,
-      "logits/chosen": -0.7278003096580505,
-      "logits/rejected": -0.48944416642189026,
-      "logps/chosen": -1177.9012451171875,
-      "logps/rejected": -1511.353271484375,
-      "loss": 0.5785,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.9513974189758301,
-      "rewards/margins": 0.3561546504497528,
-      "rewards/rejected": -1.3075522184371948,
+      "logits/chosen": -0.8570725321769714,
+      "logits/rejected": -0.636644184589386,
+      "logps/chosen": -1252.1573486328125,
+      "logps/rejected": -1564.4869384765625,
+      "loss": 0.6027,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.0256534814834595,
+      "rewards/margins": 0.33503228425979614,
+      "rewards/rejected": -1.3606857061386108,
       "step": 1100
     },
     {
       "epoch": 0.89,
       "learning_rate": 1.8863491596921745e-07,
-      "logits/chosen": -0.6377410888671875,
-      "logits/rejected": -0.3075203001499176,
-      "logps/chosen": -1318.663330078125,
-      "logps/rejected": -1740.0550537109375,
-      "loss": 0.5713,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.0838841199874878,
-      "rewards/margins": 0.48200005292892456,
-      "rewards/rejected": -1.5658843517303467,
+      "logits/chosen": -0.7760205268859863,
+      "logits/rejected": -0.4805734157562256,
+      "logps/chosen": -1414.7236328125,
+      "logps/rejected": -1822.532958984375,
+      "loss": 0.5691,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.1799442768096924,
+      "rewards/margins": 0.46841782331466675,
+      "rewards/rejected": -1.648362159729004,
       "step": 1110
     },
     {
       "epoch": 0.9,
       "learning_rate": 1.629358090099639e-07,
-      "logits/chosen": -0.6629775166511536,
-      "logits/rejected": -0.4597536623477936,
-      "logps/chosen": -1286.1126708984375,
-      "logps/rejected": -1661.913818359375,
-      "loss": 0.5857,
+      "logits/chosen": -0.7541752457618713,
+      "logits/rejected": -0.5927517414093018,
+      "logps/chosen": -1383.7718505859375,
+      "logps/rejected": -1702.1370849609375,
+      "loss": 0.5901,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.0760018825531006,
-      "rewards/margins": 0.4137555658817291,
-      "rewards/rejected": -1.4897575378417969,
+      "rewards/chosen": -1.173661231994629,
+      "rewards/margins": 0.3563198149204254,
+      "rewards/rejected": -1.529981017112732,
       "step": 1120
     },
     {
       "epoch": 0.9,
       "learning_rate": 1.3905907440629752e-07,
-      "logits/chosen": -0.6975369453430176,
-      "logits/rejected": -0.37434595823287964,
-      "logps/chosen": -1213.16357421875,
-      "logps/rejected": -1767.626953125,
-      "loss": 0.5456,
+      "logits/chosen": -0.8271347284317017,
+      "logits/rejected": -0.5372225046157837,
+      "logps/chosen": -1264.5931396484375,
+      "logps/rejected": -1810.603759765625,
+      "loss": 0.5606,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": -1.0053523778915405,
-      "rewards/margins": 0.5850921869277954,
-      "rewards/rejected": -1.590444564819336,
+      "rewards/chosen": -1.0567820072174072,
+      "rewards/margins": 0.5766392946243286,
+      "rewards/rejected": -1.633421540260315,
       "step": 1130
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.1702333051763271e-07,
-      "logits/chosen": -0.5941771268844604,
-      "logits/rejected": -0.2517922520637512,
-      "logps/chosen": -1322.6558837890625,
-      "logps/rejected": -2049.239990234375,
-      "loss": 0.5619,
+      "logits/chosen": -0.7219616770744324,
+      "logits/rejected": -0.44911837577819824,
+      "logps/chosen": -1458.1873779296875,
+      "logps/rejected": -2138.194091796875,
+      "loss": 0.5666,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.0624778270721436,
-      "rewards/margins": 0.7733040452003479,
-      "rewards/rejected": -1.8357816934585571,
+      "rewards/chosen": -1.1980092525482178,
+      "rewards/margins": 0.7267267107963562,
+      "rewards/rejected": -1.9247362613677979,
       "step": 1140
     },
     {
       "epoch": 0.92,
       "learning_rate": 9.684576015420277e-08,
-      "logits/chosen": -0.5614938139915466,
-      "logits/rejected": -0.23553869128227234,
-      "logps/chosen": -1208.92578125,
-      "logps/rejected": -1956.3076171875,
-      "loss": 0.5312,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.964518666267395,
-      "rewards/margins": 0.7551183700561523,
-      "rewards/rejected": -1.7196366786956787,
+      "logits/chosen": -0.6613628268241882,
+      "logits/rejected": -0.41062062978744507,
+      "logps/chosen": -1361.629150390625,
+      "logps/rejected": -1998.5576171875,
+      "loss": 0.5569,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.1172220706939697,
+      "rewards/margins": 0.6446647644042969,
+      "rewards/rejected": -1.7618869543075562,
       "step": 1150
     },
     {
       "epoch": 0.93,
       "learning_rate": 7.854209717842231e-08,
-      "logits/chosen": -0.47661489248275757,
-      "logits/rejected": -0.4136199951171875,
-      "logps/chosen": -1535.53125,
-      "logps/rejected": -1556.564208984375,
-      "loss": 0.5937,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.3042292594909668,
-      "rewards/margins": 0.05206754058599472,
-      "rewards/rejected": -1.3562968969345093,
+      "logits/chosen": -0.6104007959365845,
+      "logits/rejected": -0.5428999662399292,
+      "logps/chosen": -1572.9007568359375,
+      "logps/rejected": -1579.3857421875,
+      "loss": 0.5912,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3415987491607666,
+      "rewards/margins": 0.03751998022198677,
+      "rewards/rejected": -1.37911856174469,
       "step": 1160
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.212661423609184e-08,
-      "logits/chosen": -0.2740413248538971,
-      "logits/rejected": -0.08171738684177399,
-      "logps/chosen": -1799.115966796875,
-      "logps/rejected": -2178.206298828125,
-      "loss": 0.6457,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.5849744081497192,
-      "rewards/margins": 0.36505863070487976,
-      "rewards/rejected": -1.950033187866211,
+      "logits/chosen": -0.4355439245700836,
+      "logits/rejected": -0.3004533052444458,
+      "logps/chosen": -1881.150390625,
+      "logps/rejected": -2220.178955078125,
+      "loss": 0.6615,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6670087575912476,
+      "rewards/margins": 0.3249967694282532,
+      "rewards/rejected": -1.9920055866241455,
       "step": 1170
     },
     {
       "epoch": 0.94,
       "learning_rate": 4.761211162702117e-08,
-      "logits/chosen": -0.6067721247673035,
-      "logits/rejected": -0.4642402231693268,
-      "logps/chosen": -1221.586181640625,
-      "logps/rejected": -1701.192138671875,
-      "loss": 0.5565,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.9978384971618652,
-      "rewards/margins": 0.4972690939903259,
-      "rewards/rejected": -1.495107650756836,
+      "logits/chosen": -0.7097476124763489,
+      "logits/rejected": -0.6131819486618042,
+      "logps/chosen": -1349.8577880859375,
+      "logps/rejected": -1754.811279296875,
+      "loss": 0.5665,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.1261099576950073,
+      "rewards/margins": 0.42261672019958496,
+      "rewards/rejected": -1.5487267971038818,
       "step": 1180
     },
     {
       "epoch": 0.95,
       "learning_rate": 3.5009907323737826e-08,
-      "logits/chosen": -0.49088191986083984,
-      "logits/rejected": -0.08736220747232437,
-      "logps/chosen": -1518.2652587890625,
-      "logps/rejected": -2224.583251953125,
-      "loss": 0.5343,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.2854472398757935,
-      "rewards/margins": 0.7208049893379211,
-      "rewards/rejected": -2.0062522888183594,
+      "logits/chosen": -0.6964675188064575,
+      "logits/rejected": -0.35203319787979126,
+      "logps/chosen": -1499.8782958984375,
+      "logps/rejected": -2223.939697265625,
+      "loss": 0.5449,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.267060399055481,
+      "rewards/margins": 0.7385483980178833,
+      "rewards/rejected": -2.0056090354919434,
       "step": 1190
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.4329828146074096e-08,
-      "logits/chosen": -0.8410149812698364,
-      "logits/rejected": -0.44159603118896484,
-      "logps/chosen": -1532.341064453125,
-      "logps/rejected": -2004.224609375,
-      "loss": 0.5429,
+      "logits/chosen": -0.9956095814704895,
+      "logits/rejected": -0.6606365442276001,
+      "logps/chosen": -1594.940673828125,
+      "logps/rejected": -1980.383056640625,
+      "loss": 0.5542,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -1.2669811248779297,
-      "rewards/margins": 0.5341062545776367,
-      "rewards/rejected": -1.8010873794555664,
+      "rewards/chosen": -1.3295806646347046,
+      "rewards/margins": 0.44766488671302795,
+      "rewards/rejected": -1.777245283126831,
       "step": 1200
     },
     {
       "epoch": 0.97,
       "learning_rate": 1.5580202098509078e-08,
-      "logits/chosen": -0.7756412625312805,
-      "logits/rejected": -0.6899667382240295,
-      "logps/chosen": -1571.1226806640625,
-      "logps/rejected": -1902.881103515625,
-      "loss": 0.6078,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -1.2325947284698486,
-      "rewards/margins": 0.39045584201812744,
-      "rewards/rejected": -1.6230506896972656,
+      "logits/chosen": -0.9308391809463501,
+      "logits/rejected": -0.8904244303703308,
+      "logps/chosen": -1657.0823974609375,
+      "logps/rejected": -1868.926025390625,
+      "loss": 0.6277,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3185545206069946,
+      "rewards/margins": 0.27054083347320557,
+      "rewards/rejected": -1.5890953540802002,
       "step": 1210
     },
     {
       "epoch": 0.98,
       "learning_rate": 8.767851876239075e-09,
-      "logits/chosen": -0.46918898820877075,
-      "logits/rejected": -0.3102288246154785,
-      "logps/chosen": -1344.245849609375,
-      "logps/rejected": -2028.437744140625,
-      "loss": 0.5657,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -1.1186540126800537,
-      "rewards/margins": 0.6764768362045288,
-      "rewards/rejected": -1.795130729675293,
+      "logits/chosen": -0.625472903251648,
+      "logits/rejected": -0.4992523789405823,
+      "logps/chosen": -1419.646240234375,
+      "logps/rejected": -2127.7783203125,
+      "loss": 0.5692,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.194054365158081,
+      "rewards/margins": 0.7004168629646301,
+      "rewards/rejected": -1.8944714069366455,
       "step": 1220
     },
     {
       "epoch": 0.98,
       "learning_rate": 3.8980895450474455e-09,
-      "logits/chosen": -0.711412787437439,
-      "logits/rejected": -0.4764884412288666,
-      "logps/chosen": -1259.3299560546875,
-      "logps/rejected": -1813.244140625,
-      "loss": 0.6144,
+      "logits/chosen": -0.8765754699707031,
+      "logits/rejected": -0.6600515842437744,
+      "logps/chosen": -1335.862060546875,
+      "logps/rejected": -1862.95703125,
+      "loss": 0.6035,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": -1.0406744480133057,
-      "rewards/margins": 0.5738798379898071,
-      "rewards/rejected": -1.6145541667938232,
+      "rewards/chosen": -1.117206335067749,
+      "rewards/margins": 0.5470607280731201,
+      "rewards/rejected": -1.6642669439315796,
       "step": 1230
     },
     {
       "epoch": 0.99,
       "learning_rate": 9.747123991141193e-10,
-      "logits/chosen": -0.8476069569587708,
-      "logits/rejected": -0.6075129508972168,
-      "logps/chosen": -1082.3035888671875,
-      "logps/rejected": -1693.7867431640625,
-      "loss": 0.5187,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.8576550483703613,
-      "rewards/margins": 0.6194905042648315,
-      "rewards/rejected": -1.4771454334259033,
+      "logits/chosen": -0.9478354454040527,
+      "logits/rejected": -0.7755793929100037,
+      "logps/chosen": -1195.031005859375,
+      "logps/rejected": -1684.8265380859375,
+      "loss": 0.5304,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.9703825116157532,
+      "rewards/margins": 0.4978027939796448,
+      "rewards/rejected": -1.4681851863861084,
       "step": 1240
     },
     {
       "epoch": 1.0,
       "learning_rate": 0.0,
-      "logits/chosen": -0.6977601051330566,
-      "logits/rejected": -0.5082852244377136,
-      "logps/chosen": -1377.8076171875,
-      "logps/rejected": -1863.675048828125,
-      "loss": 0.5461,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -1.1340118646621704,
-      "rewards/margins": 0.5046813488006592,
-      "rewards/rejected": -1.6386932134628296,
+      "logits/chosen": -0.8340710401535034,
+      "logits/rejected": -0.6496783494949341,
+      "logps/chosen": -1408.339111328125,
+      "logps/rejected": -1972.5435791015625,
+      "loss": 0.5482,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.1645435094833374,
+      "rewards/margins": 0.58301842212677,
+      "rewards/rejected": -1.7475616931915283,
       "step": 1250
     },
     {
       "epoch": 1.0,
       "step": 1250,
       "total_flos": 0.0,
-      "train_loss": 0.6113488140106201,
-      "train_runtime": 13063.964,
-      "train_samples_per_second": 1.148,
-      "train_steps_per_second": 0.096
+      "train_loss": 0.6029718887329102,
+      "train_runtime": 12868.9286,
+      "train_samples_per_second": 1.166,
+      "train_steps_per_second": 0.097
     }
   ],
   "logging_steps": 10,