diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,4244 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 3000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.666666666666667e-08,
+      "logits/chosen": -1.8473987579345703,
+      "logits/rejected": -1.8474841117858887,
+      "logps/chosen": -628.1939086914062,
+      "logps/rejected": -1017.0994873046875,
+      "loss": 0.1606,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.6666666666666668e-07,
+      "logits/chosen": -1.4316620826721191,
+      "logits/rejected": -0.9766409397125244,
+      "logps/chosen": -518.1865234375,
+      "logps/rejected": -875.9132690429688,
+      "loss": 0.2429,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -6.813897016400006e-06,
+      "rewards/margins": 6.345117071759887e-06,
+      "rewards/rejected": -1.3159031368559226e-05,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.3333333333333335e-07,
+      "logits/chosen": -1.6879074573516846,
+      "logits/rejected": -1.2949936389923096,
+      "logps/chosen": -449.43621826171875,
+      "logps/rejected": -834.7189331054688,
+      "loss": 0.2148,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.00029412214644253254,
+      "rewards/margins": 0.00020992716599721462,
+      "rewards/rejected": 8.419497316936031e-05,
+      "step": 20
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.000000000000001e-07,
+      "logits/chosen": -1.7546484470367432,
+      "logits/rejected": -1.0084898471832275,
+      "logps/chosen": -496.73309326171875,
+      "logps/rejected": -889.1583862304688,
+      "loss": 0.2307,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.0016126197297126055,
+      "rewards/margins": 0.004062547814100981,
+      "rewards/rejected": -0.002449928317219019,
+      "step": 30
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.666666666666667e-07,
+      "logits/chosen": -1.8569256067276,
+      "logits/rejected": -1.1899112462997437,
+      "logps/chosen": -448.177490234375,
+      "logps/rejected": -852.029296875,
+      "loss": 0.2095,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.003779575927183032,
+      "rewards/margins": 0.012063580565154552,
+      "rewards/rejected": -0.008284004405140877,
+      "step": 40
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 8.333333333333333e-07,
+      "logits/chosen": -1.5573432445526123,
+      "logits/rejected": -1.0242127180099487,
+      "logps/chosen": -441.22540283203125,
+      "logps/rejected": -806.1510009765625,
+      "loss": 0.1942,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.011921902187168598,
+      "rewards/margins": 0.029057607054710388,
+      "rewards/rejected": -0.017135705798864365,
+      "step": 50
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0000000000000002e-06,
+      "logits/chosen": -1.7608438730239868,
+      "logits/rejected": -1.3249375820159912,
+      "logps/chosen": -435.998779296875,
+      "logps/rejected": -766.541015625,
+      "loss": 0.1739,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.01454467885196209,
+      "rewards/margins": 0.036728017032146454,
+      "rewards/rejected": -0.022183334454894066,
+      "step": 60
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.1666666666666668e-06,
+      "logits/chosen": -1.7882951498031616,
+      "logits/rejected": -0.9916883707046509,
+      "logps/chosen": -463.21746826171875,
+      "logps/rejected": -798.3153076171875,
+      "loss": 0.1613,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.01092723198235035,
+      "rewards/margins": 0.05515831708908081,
+      "rewards/rejected": -0.04423108696937561,
+      "step": 70
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3333333333333334e-06,
+      "logits/chosen": -1.6877580881118774,
+      "logits/rejected": -1.1066850423812866,
+      "logps/chosen": -412.07342529296875,
+      "logps/rejected": -912.7047119140625,
+      "loss": 0.1534,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.0010939339408650994,
+      "rewards/margins": 0.0824560895562172,
+      "rewards/rejected": -0.08355002105236053,
+      "step": 80
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5e-06,
+      "logits/chosen": -1.5878902673721313,
+      "logits/rejected": -1.1904621124267578,
+      "logps/chosen": -521.6202392578125,
+      "logps/rejected": -905.14794921875,
+      "loss": 0.1709,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.05081214755773544,
+      "rewards/margins": 0.08134042471647263,
+      "rewards/rejected": -0.13215258717536926,
+      "step": 90
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6666666666666667e-06,
+      "logits/chosen": -1.6791446208953857,
+      "logits/rejected": -1.182875394821167,
+      "logps/chosen": -579.6087036132812,
+      "logps/rejected": -993.3094482421875,
+      "loss": 0.149,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.09510762244462967,
+      "rewards/margins": 0.09448549151420593,
+      "rewards/rejected": -0.1895931363105774,
+      "step": 100
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8333333333333333e-06,
+      "logits/chosen": -1.6629585027694702,
+      "logits/rejected": -1.0604465007781982,
+      "logps/chosen": -638.2183837890625,
+      "logps/rejected": -1162.047119140625,
+      "loss": 0.1087,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.12550528347492218,
+      "rewards/margins": 0.14394623041152954,
+      "rewards/rejected": -0.2694514989852905,
+      "step": 110
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0000000000000003e-06,
+      "logits/chosen": -1.751172423362732,
+      "logits/rejected": -1.2760882377624512,
+      "logps/chosen": -573.3961181640625,
+      "logps/rejected": -981.23486328125,
+      "loss": 0.1234,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.09364907443523407,
+      "rewards/margins": 0.13833266496658325,
+      "rewards/rejected": -0.23198175430297852,
+      "step": 120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.166666666666667e-06,
+      "logits/chosen": -1.813155174255371,
+      "logits/rejected": -1.2474279403686523,
+      "logps/chosen": -560.3488159179688,
+      "logps/rejected": -1005.7208251953125,
+      "loss": 0.1478,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.08083701133728027,
+      "rewards/margins": 0.14707627892494202,
+      "rewards/rejected": -0.2279132902622223,
+      "step": 130
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3333333333333336e-06,
+      "logits/chosen": -1.9064534902572632,
+      "logits/rejected": -1.087799072265625,
+      "logps/chosen": -576.6889038085938,
+      "logps/rejected": -1124.29345703125,
+      "loss": 0.1001,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.11049157381057739,
+      "rewards/margins": 0.1889657974243164,
+      "rewards/rejected": -0.2994573712348938,
+      "step": 140
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.5e-06,
+      "logits/chosen": -1.498268723487854,
+      "logits/rejected": -0.9696500897407532,
+      "logps/chosen": -623.1941528320312,
+      "logps/rejected": -1241.6993408203125,
+      "loss": 0.0989,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.18307581543922424,
+      "rewards/margins": 0.2211984097957611,
+      "rewards/rejected": -0.40427422523498535,
+      "step": 150
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.666666666666667e-06,
+      "logits/chosen": -1.6134288311004639,
+      "logits/rejected": -0.7816992998123169,
+      "logps/chosen": -624.9563598632812,
+      "logps/rejected": -1204.363037109375,
+      "loss": 0.0923,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.17922857403755188,
+      "rewards/margins": 0.23642221093177795,
+      "rewards/rejected": -0.41565078496932983,
+      "step": 160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.8333333333333335e-06,
+      "logits/chosen": -1.5277217626571655,
+      "logits/rejected": -1.1026668548583984,
+      "logps/chosen": -653.5545654296875,
+      "logps/rejected": -1159.62841796875,
+      "loss": 0.0954,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.13700352609157562,
+      "rewards/margins": 0.22152018547058105,
+      "rewards/rejected": -0.35852372646331787,
+      "step": 170
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3e-06,
+      "logits/chosen": -1.6350257396697998,
+      "logits/rejected": -1.2443767786026,
+      "logps/chosen": -666.1326904296875,
+      "logps/rejected": -1370.72412109375,
+      "loss": 0.0655,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.19158519804477692,
+      "rewards/margins": 0.27321988344192505,
+      "rewards/rejected": -0.46480506658554077,
+      "step": 180
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1666666666666667e-06,
+      "logits/chosen": -1.6068347692489624,
+      "logits/rejected": -1.0125865936279297,
+      "logps/chosen": -535.7831420898438,
+      "logps/rejected": -1120.1278076171875,
+      "loss": 0.1016,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1586967408657074,
+      "rewards/margins": 0.20938825607299805,
+      "rewards/rejected": -0.36808499693870544,
+      "step": 190
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.3333333333333333e-06,
+      "logits/chosen": -1.7533979415893555,
+      "logits/rejected": -1.1328980922698975,
+      "logps/chosen": -582.9685668945312,
+      "logps/rejected": -1131.854736328125,
+      "loss": 0.0781,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.11236470937728882,
+      "rewards/margins": 0.22253580391407013,
+      "rewards/rejected": -0.33490046858787537,
+      "step": 200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5e-06,
+      "logits/chosen": -1.719315528869629,
+      "logits/rejected": -0.9589029550552368,
+      "logps/chosen": -651.3718872070312,
+      "logps/rejected": -1270.794677734375,
+      "loss": 0.1192,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1939801275730133,
+      "rewards/margins": 0.23451688885688782,
+      "rewards/rejected": -0.4284970164299011,
+      "step": 210
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.6666666666666666e-06,
+      "logits/chosen": -1.8104807138442993,
+      "logits/rejected": -1.117309808731079,
+      "logps/chosen": -525.1863403320312,
+      "logps/rejected": -1155.667724609375,
+      "loss": 0.0791,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.10046984255313873,
+      "rewards/margins": 0.22820088267326355,
+      "rewards/rejected": -0.3286706805229187,
+      "step": 220
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.833333333333334e-06,
+      "logits/chosen": -1.6729333400726318,
+      "logits/rejected": -1.3031710386276245,
+      "logps/chosen": -526.7484741210938,
+      "logps/rejected": -1042.1409912109375,
+      "loss": 0.1099,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.13879971206188202,
+      "rewards/margins": 0.17394515872001648,
+      "rewards/rejected": -0.3127448856830597,
+      "step": 230
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.000000000000001e-06,
+      "logits/chosen": -1.7155532836914062,
+      "logits/rejected": -1.3224143981933594,
+      "logps/chosen": -568.672607421875,
+      "logps/rejected": -1282.6646728515625,
+      "loss": 0.0618,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.1310817301273346,
+      "rewards/margins": 0.2731386721134186,
+      "rewards/rejected": -0.4042204022407532,
+      "step": 240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.166666666666667e-06,
+      "logits/chosen": -1.694848656654358,
+      "logits/rejected": -1.1057406663894653,
+      "logps/chosen": -608.3328247070312,
+      "logps/rejected": -1172.0714111328125,
+      "loss": 0.1015,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.1783088594675064,
+      "rewards/margins": 0.22445425391197205,
+      "rewards/rejected": -0.40276312828063965,
+      "step": 250
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.333333333333334e-06,
+      "logits/chosen": -1.4572826623916626,
+      "logits/rejected": -0.9631636738777161,
+      "logps/chosen": -678.6790161132812,
+      "logps/rejected": -1274.690185546875,
+      "loss": 0.0955,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.18504425883293152,
+      "rewards/margins": 0.24944782257080078,
+      "rewards/rejected": -0.4344921112060547,
+      "step": 260
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.5e-06,
+      "logits/chosen": -1.5380445718765259,
+      "logits/rejected": -0.9687334299087524,
+      "logps/chosen": -598.2360229492188,
+      "logps/rejected": -1067.4052734375,
+      "loss": 0.1052,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.15487895905971527,
+      "rewards/margins": 0.20970329642295837,
+      "rewards/rejected": -0.36458227038383484,
+      "step": 270
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.666666666666667e-06,
+      "logits/chosen": -1.5993750095367432,
+      "logits/rejected": -1.2316395044326782,
+      "logps/chosen": -559.2089233398438,
+      "logps/rejected": -1135.50048828125,
+      "loss": 0.0949,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.16124552488327026,
+      "rewards/margins": 0.21940717101097107,
+      "rewards/rejected": -0.38065269589424133,
+      "step": 280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.833333333333333e-06,
+      "logits/chosen": -1.785839319229126,
+      "logits/rejected": -1.2803369760513306,
+      "logps/chosen": -712.4893188476562,
+      "logps/rejected": -1194.8753662109375,
+      "loss": 0.1221,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2521515488624573,
+      "rewards/margins": 0.1936561018228531,
+      "rewards/rejected": -0.44580763578414917,
+      "step": 290
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 5e-06,
+      "logits/chosen": -1.8430877923965454,
+      "logits/rejected": -1.1291439533233643,
+      "logps/chosen": -703.2359619140625,
+      "logps/rejected": -1348.4500732421875,
+      "loss": 0.0795,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.2468656599521637,
+      "rewards/margins": 0.2653115391731262,
+      "rewards/rejected": -0.5121771693229675,
+      "step": 300
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999830770009406e-06,
+      "logits/chosen": -1.5511696338653564,
+      "logits/rejected": -0.8779551386833191,
+      "logps/chosen": -645.1519775390625,
+      "logps/rejected": -1245.902587890625,
+      "loss": 0.1052,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.16054175794124603,
+      "rewards/margins": 0.2655673027038574,
+      "rewards/rejected": -0.4261090159416199,
+      "step": 310
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999323102948655e-06,
+      "logits/chosen": -1.9738438129425049,
+      "logits/rejected": -1.4079262018203735,
+      "logps/chosen": -632.3164672851562,
+      "logps/rejected": -1090.2205810546875,
+      "loss": 0.1445,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1474119871854782,
+      "rewards/margins": 0.1809072643518448,
+      "rewards/rejected": -0.328319251537323,
+      "step": 320
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.99847706754774e-06,
+      "logits/chosen": -1.6942188739776611,
+      "logits/rejected": -1.3499834537506104,
+      "logps/chosen": -545.2244262695312,
+      "logps/rejected": -1028.6619873046875,
+      "loss": 0.0807,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.09895402193069458,
+      "rewards/margins": 0.21848241984844208,
+      "rewards/rejected": -0.31743642687797546,
+      "step": 330
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997292778346312e-06,
+      "logits/chosen": -1.8349497318267822,
+      "logits/rejected": -1.173274278640747,
+      "logps/chosen": -638.5476684570312,
+      "logps/rejected": -1342.106201171875,
+      "loss": 0.0821,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.15458893775939941,
+      "rewards/margins": 0.2725256085395813,
+      "rewards/rejected": -0.4271145761013031,
+      "step": 340
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995770395678171e-06,
+      "logits/chosen": -1.6314140558242798,
+      "logits/rejected": -1.0475269556045532,
+      "logps/chosen": -508.9056701660156,
+      "logps/rejected": -1216.2740478515625,
+      "loss": 0.0992,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.08030419796705246,
+      "rewards/margins": 0.28404951095581055,
+      "rewards/rejected": -0.3643537163734436,
+      "step": 350
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.993910125649561e-06,
+      "logits/chosen": -1.792074203491211,
+      "logits/rejected": -1.2048676013946533,
+      "logps/chosen": -606.4808959960938,
+      "logps/rejected": -1278.70703125,
+      "loss": 0.0583,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.14861299097537994,
+      "rewards/margins": 0.27981966733932495,
+      "rewards/rejected": -0.4284326136112213,
+      "step": 360
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.991712220111265e-06,
+      "logits/chosen": -1.8713674545288086,
+      "logits/rejected": -1.2337167263031006,
+      "logps/chosen": -554.6063232421875,
+      "logps/rejected": -996.7093505859375,
+      "loss": 0.1127,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.12071527540683746,
+      "rewards/margins": 0.2035401612520218,
+      "rewards/rejected": -0.32425546646118164,
+      "step": 370
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.989176976624511e-06,
+      "logits/chosen": -1.7384974956512451,
+      "logits/rejected": -1.1386005878448486,
+      "logps/chosen": -556.4752807617188,
+      "logps/rejected": -1155.403564453125,
+      "loss": 0.0799,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.13758501410484314,
+      "rewards/margins": 0.2377336472272873,
+      "rewards/rejected": -0.3753186762332916,
+      "step": 380
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.986304738420684e-06,
+      "logits/chosen": -1.8614715337753296,
+      "logits/rejected": -1.1824796199798584,
+      "logps/chosen": -628.7012329101562,
+      "logps/rejected": -1232.29296875,
+      "loss": 0.0766,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.13570387661457062,
+      "rewards/margins": 0.2571316361427307,
+      "rewards/rejected": -0.3928355276584625,
+      "step": 390
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.983095894354858e-06,
+      "logits/chosen": -1.7244367599487305,
+      "logits/rejected": -1.2317596673965454,
+      "logps/chosen": -643.6784057617188,
+      "logps/rejected": -1263.03173828125,
+      "loss": 0.0899,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.20891329646110535,
+      "rewards/margins": 0.2505062222480774,
+      "rewards/rejected": -0.4594195485115051,
+      "step": 400
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.979550878853154e-06,
+      "logits/chosen": -1.7407877445220947,
+      "logits/rejected": -0.9090331196784973,
+      "logps/chosen": -650.4755859375,
+      "logps/rejected": -1323.707763671875,
+      "loss": 0.0489,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.15291151404380798,
+      "rewards/margins": 0.3299728333950043,
+      "rewards/rejected": -0.48288434743881226,
+      "step": 410
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.975670171853926e-06,
+      "logits/chosen": -1.802283525466919,
+      "logits/rejected": -1.0241405963897705,
+      "logps/chosen": -637.089599609375,
+      "logps/rejected": -1355.4974365234375,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.1302250325679779,
+      "rewards/margins": 0.318683922290802,
+      "rewards/rejected": -0.4489089548587799,
+      "step": 420
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.971454298742779e-06,
+      "logits/chosen": -1.6149762868881226,
+      "logits/rejected": -1.1530131101608276,
+      "logps/chosen": -572.8499755859375,
+      "logps/rejected": -1171.6473388671875,
+      "loss": 0.1168,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.14183470606803894,
+      "rewards/margins": 0.26052767038345337,
+      "rewards/rejected": -0.4023623466491699,
+      "step": 430
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.966903830281449e-06,
+      "logits/chosen": -1.600284218788147,
+      "logits/rejected": -1.0997772216796875,
+      "logps/chosen": -637.7472534179688,
+      "logps/rejected": -1324.5953369140625,
+      "loss": 0.073,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.16721096634864807,
+      "rewards/margins": 0.30269843339920044,
+      "rewards/rejected": -0.4699093699455261,
+      "step": 440
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.962019382530521e-06,
+      "logits/chosen": -1.4888650178909302,
+      "logits/rejected": -0.9744981527328491,
+      "logps/chosen": -489.33538818359375,
+      "logps/rejected": -1064.3255615234375,
+      "loss": 0.1045,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.1382354497909546,
+      "rewards/margins": 0.23734359443187714,
+      "rewards/rejected": -0.3755790591239929,
+      "step": 450
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.956801616766033e-06,
+      "logits/chosen": -1.4963598251342773,
+      "logits/rejected": -1.1484572887420654,
+      "logps/chosen": -499.119384765625,
+      "logps/rejected": -1084.587158203125,
+      "loss": 0.1161,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.14311179518699646,
+      "rewards/margins": 0.24761180579662323,
+      "rewards/rejected": -0.3907236158847809,
+      "step": 460
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.951251239389949e-06,
+      "logits/chosen": -1.7252105474472046,
+      "logits/rejected": -1.16670823097229,
+      "logps/chosen": -446.770263671875,
+      "logps/rejected": -1193.3294677734375,
+      "loss": 0.0715,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.07284434884786606,
+      "rewards/margins": 0.26454949378967285,
+      "rewards/rejected": -0.3373938202857971,
+      "step": 470
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.9453690018345144e-06,
+      "logits/chosen": -1.9960441589355469,
+      "logits/rejected": -1.3124287128448486,
+      "logps/chosen": -519.2340087890625,
+      "logps/rejected": -1184.5723876953125,
+      "loss": 0.064,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.0516168549656868,
+      "rewards/margins": 0.296741783618927,
+      "rewards/rejected": -0.3483586013317108,
+      "step": 480
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.939155700460536e-06,
+      "logits/chosen": -1.7496163845062256,
+      "logits/rejected": -1.1100032329559326,
+      "logps/chosen": -532.7080078125,
+      "logps/rejected": -1199.1121826171875,
+      "loss": 0.1005,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.13813085854053497,
+      "rewards/margins": 0.2582959532737732,
+      "rewards/rejected": -0.396426796913147,
+      "step": 490
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.93261217644956e-06,
+      "logits/chosen": -1.8717424869537354,
+      "logits/rejected": -1.1161689758300781,
+      "logps/chosen": -709.0521240234375,
+      "logps/rejected": -1247.1685791015625,
+      "loss": 0.0938,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.21577858924865723,
+      "rewards/margins": 0.2319638729095459,
+      "rewards/rejected": -0.44774240255355835,
+      "step": 500
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.925739315689991e-06,
+      "logits/chosen": -1.7890201807022095,
+      "logits/rejected": -1.2021349668502808,
+      "logps/chosen": -548.541748046875,
+      "logps/rejected": -1329.703857421875,
+      "loss": 0.0815,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.11823161691427231,
+      "rewards/margins": 0.31565752625465393,
+      "rewards/rejected": -0.43388909101486206,
+      "step": 510
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.91853804865716e-06,
+      "logits/chosen": -2.0370450019836426,
+      "logits/rejected": -1.1039373874664307,
+      "logps/chosen": -516.0086669921875,
+      "logps/rejected": -1140.8289794921875,
+      "loss": 0.1045,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.031420618295669556,
+      "rewards/margins": 0.28368836641311646,
+      "rewards/rejected": -0.3151089549064636,
+      "step": 520
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.911009350287348e-06,
+      "logits/chosen": -2.096604824066162,
+      "logits/rejected": -1.4794657230377197,
+      "logps/chosen": -484.7252502441406,
+      "logps/rejected": -1161.0413818359375,
+      "loss": 0.0601,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.031848691403865814,
+      "rewards/margins": 0.29393482208251953,
+      "rewards/rejected": -0.32578352093696594,
+      "step": 530
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.903154239845798e-06,
+      "logits/chosen": -1.6835973262786865,
+      "logits/rejected": -1.0456864833831787,
+      "logps/chosen": -528.0047607421875,
+      "logps/rejected": -1127.9058837890625,
+      "loss": 0.1008,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.11576949059963226,
+      "rewards/margins": 0.23724015057086945,
+      "rewards/rejected": -0.3530096709728241,
+      "step": 540
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.894973780788722e-06,
+      "logits/chosen": -1.7007629871368408,
+      "logits/rejected": -1.2928742170333862,
+      "logps/chosen": -464.3915100097656,
+      "logps/rejected": -941.9600830078125,
+      "loss": 0.0885,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.10153596103191376,
+      "rewards/margins": 0.20636403560638428,
+      "rewards/rejected": -0.30790001153945923,
+      "step": 550
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.88646908061933e-06,
+      "logits/chosen": -1.7848955392837524,
+      "logits/rejected": -1.275628924369812,
+      "logps/chosen": -571.953369140625,
+      "logps/rejected": -1022.10595703125,
+      "loss": 0.1112,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.12155290693044662,
+      "rewards/margins": 0.1930900365114212,
+      "rewards/rejected": -0.3146429657936096,
+      "step": 560
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8776412907378845e-06,
+      "logits/chosen": -1.714331030845642,
+      "logits/rejected": -1.2135049104690552,
+      "logps/chosen": -649.1993408203125,
+      "logps/rejected": -1231.960205078125,
+      "loss": 0.0826,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.11209288984537125,
+      "rewards/margins": 0.2815489172935486,
+      "rewards/rejected": -0.39364179968833923,
+      "step": 570
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.868491606285823e-06,
+      "logits/chosen": -1.8973315954208374,
+      "logits/rejected": -1.380531907081604,
+      "logps/chosen": -590.170166015625,
+      "logps/rejected": -1119.1627197265625,
+      "loss": 0.0901,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.12205713987350464,
+      "rewards/margins": 0.2438439577817917,
+      "rewards/rejected": -0.36590105295181274,
+      "step": 580
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.859021265983959e-06,
+      "logits/chosen": -1.8590736389160156,
+      "logits/rejected": -1.3631782531738281,
+      "logps/chosen": -622.0940551757812,
+      "logps/rejected": -1332.389404296875,
+      "loss": 0.0855,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.13945552706718445,
+      "rewards/margins": 0.28879013657569885,
+      "rewards/rejected": -0.4282456338405609,
+      "step": 590
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.849231551964771e-06,
+      "logits/chosen": -1.587874174118042,
+      "logits/rejected": -1.3274825811386108,
+      "logps/chosen": -697.1937255859375,
+      "logps/rejected": -1232.4580078125,
+      "loss": 0.0898,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.2024591863155365,
+      "rewards/margins": 0.22971025109291077,
+      "rewards/rejected": -0.43216943740844727,
+      "step": 600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.839123789598829e-06,
+      "logits/chosen": -1.5191724300384521,
+      "logits/rejected": -1.2849657535552979,
+      "logps/chosen": -543.1875610351562,
+      "logps/rejected": -1340.810791015625,
+      "loss": 0.065,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.11399342864751816,
+      "rewards/margins": 0.3168831467628479,
+      "rewards/rejected": -0.43087658286094666,
+      "step": 610
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.828699347315357e-06,
+      "logits/chosen": -1.8808469772338867,
+      "logits/rejected": -1.1103613376617432,
+      "logps/chosen": -547.0265502929688,
+      "logps/rejected": -1164.1422119140625,
+      "loss": 0.0665,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.07433022558689117,
+      "rewards/margins": 0.2945881187915802,
+      "rewards/rejected": -0.36891835927963257,
+      "step": 620
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.817959636416969e-06,
+      "logits/chosen": -1.5736801624298096,
+      "logits/rejected": -0.8019571304321289,
+      "logps/chosen": -586.7124633789062,
+      "logps/rejected": -1286.5223388671875,
+      "loss": 0.076,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1401849091053009,
+      "rewards/margins": 0.3048226535320282,
+      "rewards/rejected": -0.4450075030326843,
+      "step": 630
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.806906110888606e-06,
+      "logits/chosen": -1.6860675811767578,
+      "logits/rejected": -1.2921956777572632,
+      "logps/chosen": -588.60986328125,
+      "logps/rejected": -1157.620361328125,
+      "loss": 0.0857,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.1366177797317505,
+      "rewards/margins": 0.2525423467159271,
+      "rewards/rejected": -0.3891601264476776,
+      "step": 640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.7955402672006855e-06,
+      "logits/chosen": -1.8331724405288696,
+      "logits/rejected": -1.2644875049591064,
+      "logps/chosen": -634.9255981445312,
+      "logps/rejected": -1305.0550537109375,
+      "loss": 0.084,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.14361020922660828,
+      "rewards/margins": 0.3017338216304779,
+      "rewards/rejected": -0.4453439712524414,
+      "step": 650
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.783863644106502e-06,
+      "logits/chosen": -1.7160533666610718,
+      "logits/rejected": -1.132188320159912,
+      "logps/chosen": -611.103759765625,
+      "logps/rejected": -1045.3682861328125,
+      "loss": 0.1126,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.1587737798690796,
+      "rewards/margins": 0.1975192278623581,
+      "rewards/rejected": -0.3562930226325989,
+      "step": 660
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.7718778224339115e-06,
+      "logits/chosen": -1.7684398889541626,
+      "logits/rejected": -1.1700496673583984,
+      "logps/chosen": -532.6051025390625,
+      "logps/rejected": -1173.128662109375,
+      "loss": 0.0925,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.08183015137910843,
+      "rewards/margins": 0.2822849452495575,
+      "rewards/rejected": -0.3641151189804077,
+      "step": 670
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.759584424871302e-06,
+      "logits/chosen": -1.9278764724731445,
+      "logits/rejected": -0.9649882316589355,
+      "logps/chosen": -626.4827270507812,
+      "logps/rejected": -1222.0137939453125,
+      "loss": 0.0645,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.106475330889225,
+      "rewards/margins": 0.29441386461257935,
+      "rewards/rejected": -0.40088921785354614,
+      "step": 680
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.746985115747918e-06,
+      "logits/chosen": -1.6022624969482422,
+      "logits/rejected": -1.3960731029510498,
+      "logps/chosen": -610.5640869140625,
+      "logps/rejected": -1229.7557373046875,
+      "loss": 0.0951,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.1899951994419098,
+      "rewards/margins": 0.2415318489074707,
+      "rewards/rejected": -0.4315270483493805,
+      "step": 690
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.734081600808531e-06,
+      "logits/chosen": -1.6148655414581299,
+      "logits/rejected": -1.0519297122955322,
+      "logps/chosen": -587.4749755859375,
+      "logps/rejected": -1153.4019775390625,
+      "loss": 0.0781,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.1272963136434555,
+      "rewards/margins": 0.2538054585456848,
+      "rewards/rejected": -0.3811017572879791,
+      "step": 700
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.720875626982511e-06,
+      "logits/chosen": -1.5970103740692139,
+      "logits/rejected": -1.1024632453918457,
+      "logps/chosen": -622.9388427734375,
+      "logps/rejected": -1346.39453125,
+      "loss": 0.0675,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.17155757546424866,
+      "rewards/margins": 0.3096774220466614,
+      "rewards/rejected": -0.4812350273132324,
+      "step": 710
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.707368982147318e-06,
+      "logits/chosen": -1.564493179321289,
+      "logits/rejected": -1.182403564453125,
+      "logps/chosen": -557.9251098632812,
+      "logps/rejected": -1197.14697265625,
+      "loss": 0.085,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.13098809123039246,
+      "rewards/margins": 0.27282702922821045,
+      "rewards/rejected": -0.4038150906562805,
+      "step": 720
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.693563494886455e-06,
+      "logits/chosen": -1.5731178522109985,
+      "logits/rejected": -1.2045758962631226,
+      "logps/chosen": -589.6763916015625,
+      "logps/rejected": -1163.5670166015625,
+      "loss": 0.0924,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.1421278566122055,
+      "rewards/margins": 0.26834186911582947,
+      "rewards/rejected": -0.4104697108268738,
+      "step": 730
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.679461034241906e-06,
+      "logits/chosen": -1.5476421117782593,
+      "logits/rejected": -1.2368953227996826,
+      "logps/chosen": -624.2233276367188,
+      "logps/rejected": -1224.5426025390625,
+      "loss": 0.0832,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.1516999900341034,
+      "rewards/margins": 0.26089781522750854,
+      "rewards/rejected": -0.41259780526161194,
+      "step": 740
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.665063509461098e-06,
+      "logits/chosen": -1.521240234375,
+      "logits/rejected": -0.895469069480896,
+      "logps/chosen": -604.10205078125,
+      "logps/rejected": -1188.8580322265625,
+      "loss": 0.0574,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.1560906320810318,
+      "rewards/margins": 0.2723260521888733,
+      "rewards/rejected": -0.4284166693687439,
+      "step": 750
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.650372869738415e-06,
+      "logits/chosen": -1.7068440914154053,
+      "logits/rejected": -1.068950891494751,
+      "logps/chosen": -696.2512817382812,
+      "logps/rejected": -1256.27001953125,
+      "loss": 0.0588,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.15982361137866974,
+      "rewards/margins": 0.2860753834247589,
+      "rewards/rejected": -0.44589900970458984,
+      "step": 760
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.635391103951315e-06,
+      "logits/chosen": -1.7902898788452148,
+      "logits/rejected": -1.3396075963974,
+      "logps/chosen": -573.7864990234375,
+      "logps/rejected": -1184.572998046875,
+      "loss": 0.086,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.14987358450889587,
+      "rewards/margins": 0.27520817518234253,
+      "rewards/rejected": -0.4250817894935608,
+      "step": 770
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.620120240391065e-06,
+      "logits/chosen": -1.7508132457733154,
+      "logits/rejected": -1.1751254796981812,
+      "logps/chosen": -639.8759765625,
+      "logps/rejected": -1238.5482177734375,
+      "loss": 0.0675,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.18339064717292786,
+      "rewards/margins": 0.2874487638473511,
+      "rewards/rejected": -0.47083941102027893,
+      "step": 780
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.604562346488144e-06,
+      "logits/chosen": -1.7333307266235352,
+      "logits/rejected": -1.1350371837615967,
+      "logps/chosen": -610.3933715820312,
+      "logps/rejected": -1281.648193359375,
+      "loss": 0.066,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.15978248417377472,
+      "rewards/margins": 0.2836895287036896,
+      "rewards/rejected": -0.4434719681739807,
+      "step": 790
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.588719528532342e-06,
+      "logits/chosen": -1.5033605098724365,
+      "logits/rejected": -1.08058500289917,
+      "logps/chosen": -483.33892822265625,
+      "logps/rejected": -1186.935546875,
+      "loss": 0.0829,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.12199674546718597,
+      "rewards/margins": 0.32258424162864685,
+      "rewards/rejected": -0.44458097219467163,
+      "step": 800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.572593931387604e-06,
+      "logits/chosen": -1.4822036027908325,
+      "logits/rejected": -1.1556123495101929,
+      "logps/chosen": -739.7813110351562,
+      "logps/rejected": -1216.9873046875,
+      "loss": 0.1328,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.24871714413166046,
+      "rewards/margins": 0.21633216738700867,
+      "rewards/rejected": -0.4650493264198303,
+      "step": 810
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.556187738201656e-06,
+      "logits/chosen": -1.767778754234314,
+      "logits/rejected": -1.1542942523956299,
+      "logps/chosen": -614.69677734375,
+      "logps/rejected": -1348.5400390625,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.14709432423114777,
+      "rewards/margins": 0.3316415846347809,
+      "rewards/rejected": -0.47873586416244507,
+      "step": 820
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.539503170110431e-06,
+      "logits/chosen": -1.6926486492156982,
+      "logits/rejected": -1.1664977073669434,
+      "logps/chosen": -735.2862548828125,
+      "logps/rejected": -1265.770263671875,
+      "loss": 0.0932,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.18908649682998657,
+      "rewards/margins": 0.24448461830615997,
+      "rewards/rejected": -0.4335711598396301,
+      "step": 830
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.522542485937369e-06,
+      "logits/chosen": -1.3090674877166748,
+      "logits/rejected": -1.0646418333053589,
+      "logps/chosen": -558.6735229492188,
+      "logps/rejected": -1291.3309326171875,
+      "loss": 0.073,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1368141919374466,
+      "rewards/margins": 0.3053056597709656,
+      "rewards/rejected": -0.44211989641189575,
+      "step": 840
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.50530798188761e-06,
+      "logits/chosen": -1.625422477722168,
+      "logits/rejected": -0.9810975790023804,
+      "logps/chosen": -609.5338134765625,
+      "logps/rejected": -1308.818115234375,
+      "loss": 0.0728,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.09720011800527573,
+      "rewards/margins": 0.30527880787849426,
+      "rewards/rejected": -0.4024789333343506,
+      "step": 850
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.48780199123712e-06,
+      "logits/chosen": -1.5866591930389404,
+      "logits/rejected": -1.1791462898254395,
+      "logps/chosen": -654.497314453125,
+      "logps/rejected": -1295.8955078125,
+      "loss": 0.1033,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.1393161565065384,
+      "rewards/margins": 0.2792035937309265,
+      "rewards/rejected": -0.4185197353363037,
+      "step": 860
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.470026884016805e-06,
+      "logits/chosen": -1.6597150564193726,
+      "logits/rejected": -1.071593999862671,
+      "logps/chosen": -675.8863525390625,
+      "logps/rejected": -1333.055908203125,
+      "loss": 0.0584,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.16408918797969818,
+      "rewards/margins": 0.300179660320282,
+      "rewards/rejected": -0.46426883339881897,
+      "step": 870
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.451985066691649e-06,
+      "logits/chosen": -1.533385992050171,
+      "logits/rejected": -0.9318403005599976,
+      "logps/chosen": -672.769287109375,
+      "logps/rejected": -1254.0054931640625,
+      "loss": 0.0782,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.18980053067207336,
+      "rewards/margins": 0.24227580428123474,
+      "rewards/rejected": -0.4320763647556305,
+      "step": 880
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.4336789818349105e-06,
+      "logits/chosen": -1.9593079090118408,
+      "logits/rejected": -1.2628549337387085,
+      "logps/chosen": -551.6669311523438,
+      "logps/rejected": -1144.910888671875,
+      "loss": 0.0791,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.11993531137704849,
+      "rewards/margins": 0.2696130871772766,
+      "rewards/rejected": -0.3895483613014221,
+      "step": 890
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.415111107797445e-06,
+      "logits/chosen": -1.7251970767974854,
+      "logits/rejected": -1.1421195268630981,
+      "logps/chosen": -561.43212890625,
+      "logps/rejected": -1138.663818359375,
+      "loss": 0.0933,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.08620895445346832,
+      "rewards/margins": 0.272499144077301,
+      "rewards/rejected": -0.35870811343193054,
+      "step": 900
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.396283958372173e-06,
+      "logits/chosen": -1.675585150718689,
+      "logits/rejected": -1.1132071018218994,
+      "logps/chosen": -531.9155883789062,
+      "logps/rejected": -1153.8314208984375,
+      "loss": 0.0866,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.06681212037801743,
+      "rewards/margins": 0.28186994791030884,
+      "rewards/rejected": -0.34868210554122925,
+      "step": 910
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.377200082453748e-06,
+      "logits/chosen": -1.6496893167495728,
+      "logits/rejected": -1.0332069396972656,
+      "logps/chosen": -485.20947265625,
+      "logps/rejected": -1017.17138671875,
+      "loss": 0.0822,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.057542771100997925,
+      "rewards/margins": 0.2658589780330658,
+      "rewards/rejected": -0.3234017789363861,
+      "step": 920
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.357862063693486e-06,
+      "logits/chosen": -1.699328064918518,
+      "logits/rejected": -0.9203550219535828,
+      "logps/chosen": -631.9972534179688,
+      "logps/rejected": -1354.697509765625,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.16203594207763672,
+      "rewards/margins": 0.3373655676841736,
+      "rewards/rejected": -0.4994015097618103,
+      "step": 930
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.338272520149572e-06,
+      "logits/chosen": -1.18538498878479,
+      "logits/rejected": -0.7905349731445312,
+      "logps/chosen": -693.11474609375,
+      "logps/rejected": -1318.9244384765625,
+      "loss": 0.0951,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.20799028873443604,
+      "rewards/margins": 0.2950304448604584,
+      "rewards/rejected": -0.5030207633972168,
+      "step": 940
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.318434103932622e-06,
+      "logits/chosen": -1.5692521333694458,
+      "logits/rejected": -1.009714961051941,
+      "logps/chosen": -633.1151123046875,
+      "logps/rejected": -1297.8974609375,
+      "loss": 0.0691,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.19843876361846924,
+      "rewards/margins": 0.31439024209976196,
+      "rewards/rejected": -0.5128290057182312,
+      "step": 950
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.2983495008466285e-06,
+      "logits/chosen": -1.5247453451156616,
+      "logits/rejected": -0.8545368313789368,
+      "logps/chosen": -540.1340942382812,
+      "logps/rejected": -1173.1444091796875,
+      "loss": 0.0566,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.09455517679452896,
+      "rewards/margins": 0.2999194264411926,
+      "rewards/rejected": -0.394474595785141,
+      "step": 960
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.278021430025343e-06,
+      "logits/chosen": -1.7150371074676514,
+      "logits/rejected": -0.8671354055404663,
+      "logps/chosen": -531.0009155273438,
+      "logps/rejected": -1244.0396728515625,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.11200281232595444,
+      "rewards/margins": 0.33397001028060913,
+      "rewards/rejected": -0.44597283005714417,
+      "step": 970
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.257452643564155e-06,
+      "logits/chosen": -1.7381508350372314,
+      "logits/rejected": -0.9717987179756165,
+      "logps/chosen": -660.3062744140625,
+      "logps/rejected": -1395.575927734375,
+      "loss": 0.0381,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.16878202557563782,
+      "rewards/margins": 0.35611775517463684,
+      "rewards/rejected": -0.5248997807502747,
+      "step": 980
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.236645926147493e-06,
+      "logits/chosen": -1.6484124660491943,
+      "logits/rejected": -1.1722246408462524,
+      "logps/chosen": -589.8314208984375,
+      "logps/rejected": -1167.5753173828125,
+      "loss": 0.0772,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.18166238069534302,
+      "rewards/margins": 0.2549310624599457,
+      "rewards/rejected": -0.4365933835506439,
+      "step": 990
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.215604094671835e-06,
+      "logits/chosen": -1.650120496749878,
+      "logits/rejected": -1.1718031167984009,
+      "logps/chosen": -673.8374633789062,
+      "logps/rejected": -1340.0517578125,
+      "loss": 0.0901,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.21558384597301483,
+      "rewards/margins": 0.26856911182403564,
+      "rewards/rejected": -0.4841529428958893,
+      "step": 1000
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.194329997864331e-06,
+      "logits/chosen": -1.612789511680603,
+      "logits/rejected": -1.0092923641204834,
+      "logps/chosen": -659.1913452148438,
+      "logps/rejected": -1288.3807373046875,
+      "loss": 0.0598,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.1437804251909256,
+      "rewards/margins": 0.31068724393844604,
+      "rewards/rejected": -0.45446768403053284,
+      "step": 1010
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.172826515897146e-06,
+      "logits/chosen": -1.5820199251174927,
+      "logits/rejected": -1.1678297519683838,
+      "logps/chosen": -549.3736572265625,
+      "logps/rejected": -1264.0751953125,
+      "loss": 0.0723,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.12197370827198029,
+      "rewards/margins": 0.32163676619529724,
+      "rewards/rejected": -0.4436105191707611,
+      "step": 1020
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.151096559997519e-06,
+      "logits/chosen": -1.5460295677185059,
+      "logits/rejected": -0.9339001774787903,
+      "logps/chosen": -657.1842041015625,
+      "logps/rejected": -1248.517822265625,
+      "loss": 0.0713,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.17015226185321808,
+      "rewards/margins": 0.2676629424095154,
+      "rewards/rejected": -0.43781518936157227,
+      "step": 1030
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.129143072053639e-06,
+      "logits/chosen": -1.3850871324539185,
+      "logits/rejected": -0.6763152480125427,
+      "logps/chosen": -712.0982666015625,
+      "logps/rejected": -1295.492431640625,
+      "loss": 0.0918,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.24367225170135498,
+      "rewards/margins": 0.27682679891586304,
+      "rewards/rejected": -0.5204989910125732,
+      "step": 1040
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.106969024216348e-06,
+      "logits/chosen": -1.5604678392410278,
+      "logits/rejected": -1.1585630178451538,
+      "logps/chosen": -631.67041015625,
+      "logps/rejected": -1394.1226806640625,
+      "loss": 0.0878,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.22048361599445343,
+      "rewards/margins": 0.31476694345474243,
+      "rewards/rejected": -0.5352505445480347,
+      "step": 1050
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.084577418496775e-06,
+      "logits/chosen": -1.5729732513427734,
+      "logits/rejected": -0.983799159526825,
+      "logps/chosen": -644.2813720703125,
+      "logps/rejected": -1280.1788330078125,
+      "loss": 0.067,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.21176350116729736,
+      "rewards/margins": 0.33453696966171265,
+      "rewards/rejected": -0.54630047082901,
+      "step": 1060
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.0619712863599005e-06,
+      "logits/chosen": -1.5669440031051636,
+      "logits/rejected": -1.1483709812164307,
+      "logps/chosen": -664.9472045898438,
+      "logps/rejected": -1134.748046875,
+      "loss": 0.1104,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.2157289683818817,
+      "rewards/margins": 0.22911766171455383,
+      "rewards/rejected": -0.44484663009643555,
+      "step": 1070
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.039153688314146e-06,
+      "logits/chosen": -1.594029188156128,
+      "logits/rejected": -1.106323003768921,
+      "logps/chosen": -597.1380615234375,
+      "logps/rejected": -1102.068115234375,
+      "loss": 0.1373,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17422744631767273,
+      "rewards/margins": 0.21986624598503113,
+      "rewards/rejected": -0.39409369230270386,
+      "step": 1080
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.016127713497034e-06,
+      "logits/chosen": -1.6402870416641235,
+      "logits/rejected": -0.9547089338302612,
+      "logps/chosen": -685.6248168945312,
+      "logps/rejected": -1294.0582275390625,
+      "loss": 0.0874,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.16791805624961853,
+      "rewards/margins": 0.27242207527160645,
+      "rewards/rejected": -0.440340131521225,
+      "step": 1090
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.992896479256966e-06,
+      "logits/chosen": -1.5232279300689697,
+      "logits/rejected": -1.086789608001709,
+      "logps/chosen": -680.0406494140625,
+      "logps/rejected": -1135.125,
+      "loss": 0.1149,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.22198696434497833,
+      "rewards/margins": 0.21400491893291473,
+      "rewards/rejected": -0.43599191308021545,
+      "step": 1100
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.969463130731183e-06,
+      "logits/chosen": -1.3749518394470215,
+      "logits/rejected": -0.9445620775222778,
+      "logps/chosen": -614.1600952148438,
+      "logps/rejected": -1332.16650390625,
+      "loss": 0.0712,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.16799691319465637,
+      "rewards/margins": 0.3212730288505554,
+      "rewards/rejected": -0.4892699122428894,
+      "step": 1110
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.945830840419966e-06,
+      "logits/chosen": -1.5307501554489136,
+      "logits/rejected": -0.9613451957702637,
+      "logps/chosen": -572.4707641601562,
+      "logps/rejected": -1237.0767822265625,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.14543718099594116,
+      "rewards/margins": 0.29924580454826355,
+      "rewards/rejected": -0.4446829855442047,
+      "step": 1120
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.92200280775713e-06,
+      "logits/chosen": -1.3332005739212036,
+      "logits/rejected": -0.8951355218887329,
+      "logps/chosen": -686.0225830078125,
+      "logps/rejected": -1341.870361328125,
+      "loss": 0.0835,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.22055836021900177,
+      "rewards/margins": 0.28799134492874146,
+      "rewards/rejected": -0.508549690246582,
+      "step": 1130
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.897982258676867e-06,
+      "logits/chosen": -1.2923948764801025,
+      "logits/rejected": -1.0148731470108032,
+      "logps/chosen": -460.26336669921875,
+      "logps/rejected": -1217.972412109375,
+      "loss": 0.067,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.09934146702289581,
+      "rewards/margins": 0.30596452951431274,
+      "rewards/rejected": -0.40530601143836975,
+      "step": 1140
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.8737724451770155e-06,
+      "logits/chosen": -1.660675287246704,
+      "logits/rejected": -0.7988176345825195,
+      "logps/chosen": -664.8593139648438,
+      "logps/rejected": -1191.18017578125,
+      "loss": 0.0708,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.12432744354009628,
+      "rewards/margins": 0.2866395115852356,
+      "rewards/rejected": -0.4109669625759125,
+      "step": 1150
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.849376644878783e-06,
+      "logits/chosen": -1.7713876962661743,
+      "logits/rejected": -0.9808756113052368,
+      "logps/chosen": -621.3865356445312,
+      "logps/rejected": -1367.19482421875,
+      "loss": 0.0447,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.1509043574333191,
+      "rewards/margins": 0.34973520040512085,
+      "rewards/rejected": -0.5006396174430847,
+      "step": 1160
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.824798160583012e-06,
+      "logits/chosen": -1.5198707580566406,
+      "logits/rejected": -0.90589439868927,
+      "logps/chosen": -679.7938232421875,
+      "logps/rejected": -1217.0797119140625,
+      "loss": 0.0979,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.20723366737365723,
+      "rewards/margins": 0.2676594853401184,
+      "rewards/rejected": -0.47489315271377563,
+      "step": 1170
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8000403198230385e-06,
+      "logits/chosen": -1.8150522708892822,
+      "logits/rejected": -0.9454204440116882,
+      "logps/chosen": -640.8989868164062,
+      "logps/rejected": -1308.7498779296875,
+      "loss": 0.0713,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.12217221409082413,
+      "rewards/margins": 0.3217760920524597,
+      "rewards/rejected": -0.44394832849502563,
+      "step": 1180
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7751064744141886e-06,
+      "logits/chosen": -1.1891100406646729,
+      "logits/rejected": -1.013519525527954,
+      "logps/chosen": -462.0289001464844,
+      "logps/rejected": -1144.2987060546875,
+      "loss": 0.0713,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.057515986263751984,
+      "rewards/margins": 0.3108125925064087,
+      "rewards/rejected": -0.36832860112190247,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7500000000000005e-06,
+      "logits/chosen": -1.53715980052948,
+      "logits/rejected": -1.0485864877700806,
+      "logps/chosen": -620.6614379882812,
+      "logps/rejected": -1361.7210693359375,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.14007523655891418,
+      "rewards/margins": 0.3020576536655426,
+      "rewards/rejected": -0.4421328604221344,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.724724295595218e-06,
+      "logits/chosen": -1.4643341302871704,
+      "logits/rejected": -0.7922446131706238,
+      "logps/chosen": -477.1775817871094,
+      "logps/rejected": -1089.298583984375,
+      "loss": 0.0713,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.0700678676366806,
+      "rewards/margins": 0.27943548560142517,
+      "rewards/rejected": -0.3495033383369446,
+      "step": 1210
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.699282783125616e-06,
+      "logits/chosen": -1.6988475322723389,
+      "logits/rejected": -1.08052396774292,
+      "logps/chosen": -519.7030029296875,
+      "logps/rejected": -1169.624755859375,
+      "loss": 0.0815,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.10375889390707016,
+      "rewards/margins": 0.2844247817993164,
+      "rewards/rejected": -0.3881836533546448,
+      "step": 1220
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6736789069647273e-06,
+      "logits/chosen": -1.5467779636383057,
+      "logits/rejected": -0.878455638885498,
+      "logps/chosen": -689.0154418945312,
+      "logps/rejected": -1260.757568359375,
+      "loss": 0.0964,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.20392462611198425,
+      "rewards/margins": 0.26304885745048523,
+      "rewards/rejected": -0.46697354316711426,
+      "step": 1230
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6479161334675294e-06,
+      "logits/chosen": -1.4715616703033447,
+      "logits/rejected": -0.9548036456108093,
+      "logps/chosen": -503.00982666015625,
+      "logps/rejected": -1227.5513916015625,
+      "loss": 0.06,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.09338453412055969,
+      "rewards/margins": 0.32899850606918335,
+      "rewards/rejected": -0.42238301038742065,
+      "step": 1240
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.621997950501156e-06,
+      "logits/chosen": -1.564557433128357,
+      "logits/rejected": -1.0119483470916748,
+      "logps/chosen": -533.0374145507812,
+      "logps/rejected": -1133.299072265625,
+      "loss": 0.0832,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.12650765478610992,
+      "rewards/margins": 0.2633581757545471,
+      "rewards/rejected": -0.38986584544181824,
+      "step": 1250
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.595927866972694e-06,
+      "logits/chosen": -1.5929651260375977,
+      "logits/rejected": -0.946239173412323,
+      "logps/chosen": -796.1531982421875,
+      "logps/rejected": -1373.961181640625,
+      "loss": 0.1067,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.258171945810318,
+      "rewards/margins": 0.25018230080604553,
+      "rewards/rejected": -0.5083541870117188,
+      "step": 1260
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5697094123541357e-06,
+      "logits/chosen": -1.3169076442718506,
+      "logits/rejected": -0.6554785370826721,
+      "logps/chosen": -671.5099487304688,
+      "logps/rejected": -1274.65576171875,
+      "loss": 0.072,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.23343141376972198,
+      "rewards/margins": 0.258681058883667,
+      "rewards/rejected": -0.49211248755455017,
+      "step": 1270
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.543346136204545e-06,
+      "logits/chosen": -1.5627593994140625,
+      "logits/rejected": -1.0817673206329346,
+      "logps/chosen": -626.169189453125,
+      "logps/rejected": -1331.075439453125,
+      "loss": 0.0776,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.23857741057872772,
+      "rewards/margins": 0.29800939559936523,
+      "rewards/rejected": -0.5365867614746094,
+      "step": 1280
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.516841607689501e-06,
+      "logits/chosen": -1.4650442600250244,
+      "logits/rejected": -0.9328089952468872,
+      "logps/chosen": -621.0172729492188,
+      "logps/rejected": -1351.593017578125,
+      "loss": 0.0719,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.204871267080307,
+      "rewards/margins": 0.3157797157764435,
+      "rewards/rejected": -0.5206509828567505,
+      "step": 1290
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.4901994150978926e-06,
+      "logits/chosen": -1.4772984981536865,
+      "logits/rejected": -1.0536553859710693,
+      "logps/chosen": -532.893310546875,
+      "logps/rejected": -1081.2454833984375,
+      "loss": 0.0756,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.09208095073699951,
+      "rewards/margins": 0.2642812132835388,
+      "rewards/rejected": -0.35636216402053833,
+      "step": 1300
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4634231653561213e-06,
+      "logits/chosen": -1.5390266180038452,
+      "logits/rejected": -0.9685665965080261,
+      "logps/chosen": -569.3391723632812,
+      "logps/rejected": -1077.9581298828125,
+      "loss": 0.0826,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.08572982251644135,
+      "rewards/margins": 0.25851181149482727,
+      "rewards/rejected": -0.3442416191101074,
+      "step": 1310
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.436516483539781e-06,
+      "logits/chosen": -1.7010656595230103,
+      "logits/rejected": -1.0056097507476807,
+      "logps/chosen": -629.4781494140625,
+      "logps/rejected": -1221.6732177734375,
+      "loss": 0.0792,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.13440915942192078,
+      "rewards/margins": 0.266291081905365,
+      "rewards/rejected": -0.40070027112960815,
+      "step": 1320
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.409483012382879e-06,
+      "logits/chosen": -1.8139673471450806,
+      "logits/rejected": -1.0937250852584839,
+      "logps/chosen": -542.4208984375,
+      "logps/rejected": -1275.978271484375,
+      "loss": 0.0647,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.12924353778362274,
+      "rewards/margins": 0.3123559355735779,
+      "rewards/rejected": -0.44159942865371704,
+      "step": 1330
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3823264117846722e-06,
+      "logits/chosen": -1.34089195728302,
+      "logits/rejected": -1.0331387519836426,
+      "logps/chosen": -669.740966796875,
+      "logps/rejected": -1183.39404296875,
+      "loss": 0.0927,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.16455522179603577,
+      "rewards/margins": 0.24129804968833923,
+      "rewards/rejected": -0.4058533310890198,
+      "step": 1340
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3550503583141726e-06,
+      "logits/chosen": -1.4803988933563232,
+      "logits/rejected": -1.2195205688476562,
+      "logps/chosen": -631.5969848632812,
+      "logps/rejected": -1318.831787109375,
+      "loss": 0.0847,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.24165017902851105,
+      "rewards/margins": 0.2722802758216858,
+      "rewards/rejected": -0.5139304399490356,
+      "step": 1350
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3276585447123957e-06,
+      "logits/chosen": -1.698537826538086,
+      "logits/rejected": -1.1106715202331543,
+      "logps/chosen": -758.1787719726562,
+      "logps/rejected": -1358.7789306640625,
+      "loss": 0.0781,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.2571624517440796,
+      "rewards/margins": 0.247967928647995,
+      "rewards/rejected": -0.5051303505897522,
+      "step": 1360
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.300154679392429e-06,
+      "logits/chosen": -1.565079927444458,
+      "logits/rejected": -0.8933359980583191,
+      "logps/chosen": -655.6455078125,
+      "logps/rejected": -1246.6724853515625,
+      "loss": 0.0855,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.2172234058380127,
+      "rewards/margins": 0.259296715259552,
+      "rewards/rejected": -0.4765201508998871,
+      "step": 1370
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.272542485937369e-06,
+      "logits/chosen": -1.7766815423965454,
+      "logits/rejected": -1.0995599031448364,
+      "logps/chosen": -622.3171997070312,
+      "logps/rejected": -1246.709716796875,
+      "loss": 0.0775,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.16058366000652313,
+      "rewards/margins": 0.2945863604545593,
+      "rewards/rejected": -0.45517000555992126,
+      "step": 1380
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.244825702596205e-06,
+      "logits/chosen": -1.6319786310195923,
+      "logits/rejected": -1.2775378227233887,
+      "logps/chosen": -543.9849853515625,
+      "logps/rejected": -1321.574462890625,
+      "loss": 0.0694,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.13428989052772522,
+      "rewards/margins": 0.3316313922405243,
+      "rewards/rejected": -0.4659213125705719,
+      "step": 1390
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.217008081777726e-06,
+      "logits/chosen": -1.6392805576324463,
+      "logits/rejected": -1.2436825037002563,
+      "logps/chosen": -662.4228515625,
+      "logps/rejected": -1163.439208984375,
+      "loss": 0.1084,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.1475670039653778,
+      "rewards/margins": 0.2356584370136261,
+      "rewards/rejected": -0.38322538137435913,
+      "step": 1400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.189093389542498e-06,
+      "logits/chosen": -1.8173353672027588,
+      "logits/rejected": -0.9919673204421997,
+      "logps/chosen": -665.50537109375,
+      "logps/rejected": -1430.367431640625,
+      "loss": 0.0367,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.16980603337287903,
+      "rewards/margins": 0.3760393559932709,
+      "rewards/rejected": -0.5458453893661499,
+      "step": 1410
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1610854050930063e-06,
+      "logits/chosen": -1.5377254486083984,
+      "logits/rejected": -0.8976453542709351,
+      "logps/chosen": -662.4138793945312,
+      "logps/rejected": -1309.8470458984375,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.20192308723926544,
+      "rewards/margins": 0.30307039618492126,
+      "rewards/rejected": -0.5049934983253479,
+      "step": 1420
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.132987920262005e-06,
+      "logits/chosen": -1.5729992389678955,
+      "logits/rejected": -0.892600417137146,
+      "logps/chosen": -780.0401000976562,
+      "logps/rejected": -1402.983154296875,
+      "loss": 0.0667,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.18477649986743927,
+      "rewards/margins": 0.33369770646095276,
+      "rewards/rejected": -0.5184742212295532,
+      "step": 1430
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1048047389991693e-06,
+      "logits/chosen": -1.5943243503570557,
+      "logits/rejected": -0.9319471120834351,
+      "logps/chosen": -730.2401733398438,
+      "logps/rejected": -1355.317626953125,
+      "loss": 0.0786,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1811259537935257,
+      "rewards/margins": 0.3043223023414612,
+      "rewards/rejected": -0.48544827103614807,
+      "step": 1440
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0765396768561005e-06,
+      "logits/chosen": -1.8524019718170166,
+      "logits/rejected": -0.9115484356880188,
+      "logps/chosen": -643.4202270507812,
+      "logps/rejected": -1369.478271484375,
+      "loss": 0.0631,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.1561841517686844,
+      "rewards/margins": 0.35857564210891724,
+      "rewards/rejected": -0.5147597193717957,
+      "step": 1450
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0481965604697582e-06,
+      "logits/chosen": -1.8506492376327515,
+      "logits/rejected": -1.3686341047286987,
+      "logps/chosen": -625.1241455078125,
+      "logps/rejected": -1145.7611083984375,
+      "loss": 0.0856,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.14623615145683289,
+      "rewards/margins": 0.25150930881500244,
+      "rewards/rejected": -0.39774543046951294,
+      "step": 1460
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.019779227044398e-06,
+      "logits/chosen": -1.5452914237976074,
+      "logits/rejected": -1.0970354080200195,
+      "logps/chosen": -576.4892578125,
+      "logps/rejected": -1214.838623046875,
+      "loss": 0.054,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.15811112523078918,
+      "rewards/margins": 0.29070132970809937,
+      "rewards/rejected": -0.44881248474121094,
+      "step": 1470
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.9912915238320755e-06,
+      "logits/chosen": -1.5780893564224243,
+      "logits/rejected": -1.1927622556686401,
+      "logps/chosen": -574.8433227539062,
+      "logps/rejected": -1284.197998046875,
+      "loss": 0.0639,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.14699247479438782,
+      "rewards/margins": 0.30907437205314636,
+      "rewards/rejected": -0.4560668468475342,
+      "step": 1480
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9627373076117864e-06,
+      "logits/chosen": -1.6259567737579346,
+      "logits/rejected": -1.0762062072753906,
+      "logps/chosen": -532.4714965820312,
+      "logps/rejected": -1246.502685546875,
+      "loss": 0.0554,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.15746064484119415,
+      "rewards/margins": 0.3116622865200043,
+      "rewards/rejected": -0.4691229462623596,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9341204441673267e-06,
+      "logits/chosen": -1.4439929723739624,
+      "logits/rejected": -1.0250179767608643,
+      "logps/chosen": -622.338134765625,
+      "logps/rejected": -1203.972412109375,
+      "loss": 0.0955,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.17773118615150452,
+      "rewards/margins": 0.2580625116825104,
+      "rewards/rejected": -0.4357937276363373,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9054448077639193e-06,
+      "logits/chosen": -1.5762206315994263,
+      "logits/rejected": -0.9138490557670593,
+      "logps/chosen": -586.79345703125,
+      "logps/rejected": -1128.914794921875,
+      "loss": 0.1417,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.16046829521656036,
+      "rewards/margins": 0.23752140998840332,
+      "rewards/rejected": -0.3979896903038025,
+      "step": 1510
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.876714280623708e-06,
+      "logits/chosen": -1.5804052352905273,
+      "logits/rejected": -1.3289867639541626,
+      "logps/chosen": -565.5819091796875,
+      "logps/rejected": -1220.959228515625,
+      "loss": 0.0819,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.16751018166542053,
+      "rewards/margins": 0.2462599277496338,
+      "rewards/rejected": -0.4137701392173767,
+      "step": 1520
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.847932752400164e-06,
+      "logits/chosen": -1.6598694324493408,
+      "logits/rejected": -1.195133090019226,
+      "logps/chosen": -587.2601318359375,
+      "logps/rejected": -1277.15283203125,
+      "loss": 0.0684,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.12616164982318878,
+      "rewards/margins": 0.29237666726112366,
+      "rewards/rejected": -0.41853827238082886,
+      "step": 1530
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8191041196514874e-06,
+      "logits/chosen": -1.658347725868225,
+      "logits/rejected": -0.909757137298584,
+      "logps/chosen": -614.3128662109375,
+      "logps/rejected": -1271.3431396484375,
+      "loss": 0.0714,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.14127101004123688,
+      "rewards/margins": 0.2985311448574066,
+      "rewards/rejected": -0.43980222940444946,
+      "step": 1540
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7902322853130758e-06,
+      "logits/chosen": -1.7248843908309937,
+      "logits/rejected": -1.4133408069610596,
+      "logps/chosen": -530.6253662109375,
+      "logps/rejected": -1091.8160400390625,
+      "loss": 0.0698,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.12663719058036804,
+      "rewards/margins": 0.252753883600235,
+      "rewards/rejected": -0.3793911337852478,
+      "step": 1550
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.761321158169134e-06,
+      "logits/chosen": -1.8168293237686157,
+      "logits/rejected": -1.3112378120422363,
+      "logps/chosen": -532.3922119140625,
+      "logps/rejected": -1184.728515625,
+      "loss": 0.065,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.11639068275690079,
+      "rewards/margins": 0.30524441599845886,
+      "rewards/rejected": -0.42163509130477905,
+      "step": 1560
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7323746523234813e-06,
+      "logits/chosen": -1.6596038341522217,
+      "logits/rejected": -1.0099501609802246,
+      "logps/chosen": -620.3221435546875,
+      "logps/rejected": -1408.0771484375,
+      "loss": 0.035,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.11354754865169525,
+      "rewards/margins": 0.3631085455417633,
+      "rewards/rejected": -0.47665613889694214,
+      "step": 1570
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.703396686669646e-06,
+      "logits/chosen": -1.677464246749878,
+      "logits/rejected": -1.2843691110610962,
+      "logps/chosen": -601.0008544921875,
+      "logps/rejected": -1275.202880859375,
+      "loss": 0.0639,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.11773763597011566,
+      "rewards/margins": 0.30820992588996887,
+      "rewards/rejected": -0.4259475767612457,
+      "step": 1580
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6743911843603134e-06,
+      "logits/chosen": -1.915787696838379,
+      "logits/rejected": -0.9031229019165039,
+      "logps/chosen": -596.1331176757812,
+      "logps/rejected": -1234.94482421875,
+      "loss": 0.0562,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.11484452337026596,
+      "rewards/margins": 0.32129010558128357,
+      "rewards/rejected": -0.4361346364021301,
+      "step": 1590
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6453620722761897e-06,
+      "logits/chosen": -1.800848364830017,
+      "logits/rejected": -1.0119121074676514,
+      "logps/chosen": -777.0930786132812,
+      "logps/rejected": -1372.353271484375,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.13788454234600067,
+      "rewards/margins": 0.31275519728660583,
+      "rewards/rejected": -0.4506397843360901,
+      "step": 1600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6163132804943675e-06,
+      "logits/chosen": -1.7606815099716187,
+      "logits/rejected": -1.1106458902359009,
+      "logps/chosen": -619.3434448242188,
+      "logps/rejected": -1193.679931640625,
+      "loss": 0.0785,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.1508094221353531,
+      "rewards/margins": 0.26593416929244995,
+      "rewards/rejected": -0.41674357652664185,
+      "step": 1610
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.587248741756253e-06,
+      "logits/chosen": -1.8017308712005615,
+      "logits/rejected": -1.0231897830963135,
+      "logps/chosen": -573.2968139648438,
+      "logps/rejected": -1028.872802734375,
+      "loss": 0.0863,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.14274278283119202,
+      "rewards/margins": 0.22178173065185547,
+      "rewards/rejected": -0.3645245134830475,
+      "step": 1620
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.558172390935141e-06,
+      "logits/chosen": -1.8495060205459595,
+      "logits/rejected": -0.9550938606262207,
+      "logps/chosen": -701.3528442382812,
+      "logps/rejected": -1321.211181640625,
+      "loss": 0.0576,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.20430326461791992,
+      "rewards/margins": 0.292900025844574,
+      "rewards/rejected": -0.4972032606601715,
+      "step": 1630
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5290881645034932e-06,
+      "logits/chosen": -1.613519310951233,
+      "logits/rejected": -1.0299922227859497,
+      "logps/chosen": -682.4573974609375,
+      "logps/rejected": -1327.379150390625,
+      "loss": 0.0725,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.2211301326751709,
+      "rewards/margins": 0.2834627032279968,
+      "rewards/rejected": -0.5045928359031677,
+      "step": 1640
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5e-06,
+      "logits/chosen": -1.8198554515838623,
+      "logits/rejected": -1.2243192195892334,
+      "logps/chosen": -673.7959594726562,
+      "logps/rejected": -1209.7906494140625,
+      "loss": 0.0879,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.15559658408164978,
+      "rewards/margins": 0.2655668258666992,
+      "rewards/rejected": -0.4211633801460266,
+      "step": 1650
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.470911835496508e-06,
+      "logits/chosen": -1.6637722253799438,
+      "logits/rejected": -1.15647292137146,
+      "logps/chosen": -611.1282348632812,
+      "logps/rejected": -1223.1854248046875,
+      "loss": 0.0801,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.1450941264629364,
+      "rewards/margins": 0.2773655652999878,
+      "rewards/rejected": -0.4224596917629242,
+      "step": 1660
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4418276090648596e-06,
+      "logits/chosen": -1.8464033603668213,
+      "logits/rejected": -1.052440881729126,
+      "logps/chosen": -644.9616088867188,
+      "logps/rejected": -1258.923095703125,
+      "loss": 0.0874,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1209816113114357,
+      "rewards/margins": 0.2808608412742615,
+      "rewards/rejected": -0.40184250473976135,
+      "step": 1670
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4127512582437486e-06,
+      "logits/chosen": -1.685725212097168,
+      "logits/rejected": -1.0185719728469849,
+      "logps/chosen": -620.1656494140625,
+      "logps/rejected": -1276.2811279296875,
+      "loss": 0.0583,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.14368347823619843,
+      "rewards/margins": 0.2920317053794861,
+      "rewards/rejected": -0.4357151985168457,
+      "step": 1680
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3836867195056333e-06,
+      "logits/chosen": -1.6978428363800049,
+      "logits/rejected": -1.209702730178833,
+      "logps/chosen": -633.7965698242188,
+      "logps/rejected": -1311.2330322265625,
+      "loss": 0.0559,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.15191814303398132,
+      "rewards/margins": 0.2970329523086548,
+      "rewards/rejected": -0.4489511549472809,
+      "step": 1690
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3546379277238107e-06,
+      "logits/chosen": -1.4833507537841797,
+      "logits/rejected": -1.0119761228561401,
+      "logps/chosen": -680.2658081054688,
+      "logps/rejected": -1296.93212890625,
+      "loss": 0.0718,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.23375864326953888,
+      "rewards/margins": 0.2688060402870178,
+      "rewards/rejected": -0.5025646686553955,
+      "step": 1700
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.325608815639687e-06,
+      "logits/chosen": -1.6255954504013062,
+      "logits/rejected": -0.9894803166389465,
+      "logps/chosen": -608.648681640625,
+      "logps/rejected": -1257.4925537109375,
+      "loss": 0.0806,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.19161316752433777,
+      "rewards/margins": 0.27543801069259644,
+      "rewards/rejected": -0.46705111861228943,
+      "step": 1710
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.296603313330355e-06,
+      "logits/chosen": -1.3378230333328247,
+      "logits/rejected": -0.8268178105354309,
+      "logps/chosen": -624.6492309570312,
+      "logps/rejected": -1211.5384521484375,
+      "loss": 0.1075,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.17298360168933868,
+      "rewards/margins": 0.25485867261886597,
+      "rewards/rejected": -0.42784222960472107,
+      "step": 1720
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.26762534767652e-06,
+      "logits/chosen": -1.7436068058013916,
+      "logits/rejected": -1.0120248794555664,
+      "logps/chosen": -667.2508544921875,
+      "logps/rejected": -1316.541015625,
+      "loss": 0.0723,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.18062946200370789,
+      "rewards/margins": 0.2949804961681366,
+      "rewards/rejected": -0.47561001777648926,
+      "step": 1730
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.238678841830867e-06,
+      "logits/chosen": -1.743772268295288,
+      "logits/rejected": -0.9547621607780457,
+      "logps/chosen": -719.8886108398438,
+      "logps/rejected": -1386.740478515625,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.1676359474658966,
+      "rewards/margins": 0.31254929304122925,
+      "rewards/rejected": -0.48018521070480347,
+      "step": 1740
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2097677146869242e-06,
+      "logits/chosen": -1.7221221923828125,
+      "logits/rejected": -1.1789166927337646,
+      "logps/chosen": -515.1943359375,
+      "logps/rejected": -1109.064453125,
+      "loss": 0.0696,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.11262670904397964,
+      "rewards/margins": 0.2698899805545807,
+      "rewards/rejected": -0.38251665234565735,
+      "step": 1750
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1808958803485134e-06,
+      "logits/chosen": -1.7244141101837158,
+      "logits/rejected": -1.0081803798675537,
+      "logps/chosen": -617.8436889648438,
+      "logps/rejected": -1301.739990234375,
+      "loss": 0.0449,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.13489536941051483,
+      "rewards/margins": 0.30581116676330566,
+      "rewards/rejected": -0.4407065510749817,
+      "step": 1760
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1520672475998374e-06,
+      "logits/chosen": -1.5150662660598755,
+      "logits/rejected": -1.2830677032470703,
+      "logps/chosen": -469.53497314453125,
+      "logps/rejected": -1185.6640625,
+      "loss": 0.1188,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.12466877698898315,
+      "rewards/margins": 0.25860312581062317,
+      "rewards/rejected": -0.3832719027996063,
+      "step": 1770
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1232857193762923e-06,
+      "logits/chosen": -1.6666500568389893,
+      "logits/rejected": -1.1478335857391357,
+      "logps/chosen": -630.8321533203125,
+      "logps/rejected": -1219.052978515625,
+      "loss": 0.0988,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.14840959012508392,
+      "rewards/margins": 0.2695569396018982,
+      "rewards/rejected": -0.4179665148258209,
+      "step": 1780
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.094555192236082e-06,
+      "logits/chosen": -1.6702663898468018,
+      "logits/rejected": -1.1331311464309692,
+      "logps/chosen": -639.830322265625,
+      "logps/rejected": -1440.0772705078125,
+      "loss": 0.0343,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.16856852173805237,
+      "rewards/margins": 0.34757930040359497,
+      "rewards/rejected": -0.5161478519439697,
+      "step": 1790
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0658795558326745e-06,
+      "logits/chosen": -1.3389908075332642,
+      "logits/rejected": -1.1332718133926392,
+      "logps/chosen": -560.32763671875,
+      "logps/rejected": -1140.04541015625,
+      "loss": 0.0962,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.18182942271232605,
+      "rewards/margins": 0.22485823929309845,
+      "rewards/rejected": -0.4066876769065857,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.037262692388214e-06,
+      "logits/chosen": -1.5946425199508667,
+      "logits/rejected": -1.050821304321289,
+      "logps/chosen": -601.1361083984375,
+      "logps/rejected": -1288.375244140625,
+      "loss": 0.0656,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.13252010941505432,
+      "rewards/margins": 0.3152475953102112,
+      "rewards/rejected": -0.4477676749229431,
+      "step": 1810
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0087084761679245e-06,
+      "logits/chosen": -1.9240398406982422,
+      "logits/rejected": -1.0551602840423584,
+      "logps/chosen": -725.8641967773438,
+      "logps/rejected": -1185.0716552734375,
+      "loss": 0.0752,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1920987367630005,
+      "rewards/margins": 0.24843771755695343,
+      "rewards/rejected": -0.4405364394187927,
+      "step": 1820
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9802207729556023e-06,
+      "logits/chosen": -1.827103853225708,
+      "logits/rejected": -1.2141703367233276,
+      "logps/chosen": -591.169677734375,
+      "logps/rejected": -1252.523193359375,
+      "loss": 0.0783,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.18946315348148346,
+      "rewards/margins": 0.2804568111896515,
+      "rewards/rejected": -0.46991997957229614,
+      "step": 1830
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9518034395302413e-06,
+      "logits/chosen": -1.4736969470977783,
+      "logits/rejected": -0.9016133546829224,
+      "logps/chosen": -606.1239013671875,
+      "logps/rejected": -1232.34912109375,
+      "loss": 0.0613,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.16971653699874878,
+      "rewards/margins": 0.3104923367500305,
+      "rewards/rejected": -0.4802088141441345,
+      "step": 1840
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9234603231439e-06,
+      "logits/chosen": -1.7311079502105713,
+      "logits/rejected": -1.0143775939941406,
+      "logps/chosen": -625.5157470703125,
+      "logps/rejected": -1218.10693359375,
+      "loss": 0.0795,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.12972071766853333,
+      "rewards/margins": 0.30200275778770447,
+      "rewards/rejected": -0.4317234456539154,
+      "step": 1850
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.895195261000831e-06,
+      "logits/chosen": -1.702005386352539,
+      "logits/rejected": -1.2821683883666992,
+      "logps/chosen": -580.2069091796875,
+      "logps/rejected": -1129.827880859375,
+      "loss": 0.092,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.12076668441295624,
+      "rewards/margins": 0.2600669264793396,
+      "rewards/rejected": -0.38083356618881226,
+      "step": 1860
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8670120797379958e-06,
+      "logits/chosen": -1.5771738290786743,
+      "logits/rejected": -1.2320917844772339,
+      "logps/chosen": -590.98046875,
+      "logps/rejected": -1251.115234375,
+      "loss": 0.1923,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.14493119716644287,
+      "rewards/margins": 0.3147684633731842,
+      "rewards/rejected": -0.4596996307373047,
+      "step": 1870
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8389145949069953e-06,
+      "logits/chosen": -1.7696335315704346,
+      "logits/rejected": -1.096189260482788,
+      "logps/chosen": -578.0641479492188,
+      "logps/rejected": -1174.998291015625,
+      "loss": 0.0784,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.13082124292850494,
+      "rewards/margins": 0.29184988141059875,
+      "rewards/rejected": -0.4226710796356201,
+      "step": 1880
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8109066104575023e-06,
+      "logits/chosen": -1.5320732593536377,
+      "logits/rejected": -0.7170606851577759,
+      "logps/chosen": -602.5262451171875,
+      "logps/rejected": -1216.552978515625,
+      "loss": 0.0593,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.13688640296459198,
+      "rewards/margins": 0.3167448043823242,
+      "rewards/rejected": -0.4536312222480774,
+      "step": 1890
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7829919182222752e-06,
+      "logits/chosen": -1.6282037496566772,
+      "logits/rejected": -0.7399900555610657,
+      "logps/chosen": -665.1109008789062,
+      "logps/rejected": -1210.0506591796875,
+      "loss": 0.0832,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.17592889070510864,
+      "rewards/margins": 0.27498775720596313,
+      "rewards/rejected": -0.4509166181087494,
+      "step": 1900
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.755174297403795e-06,
+      "logits/chosen": -1.5313676595687866,
+      "logits/rejected": -1.0872125625610352,
+      "logps/chosen": -673.0841064453125,
+      "logps/rejected": -1284.4078369140625,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.22001402080059052,
+      "rewards/margins": 0.2925686240196228,
+      "rewards/rejected": -0.5125826597213745,
+      "step": 1910
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7274575140626318e-06,
+      "logits/chosen": -1.5627118349075317,
+      "logits/rejected": -0.9476549029350281,
+      "logps/chosen": -889.22509765625,
+      "logps/rejected": -1250.838134765625,
+      "loss": 0.1224,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.24946339428424835,
+      "rewards/margins": 0.240512415766716,
+      "rewards/rejected": -0.4899757504463196,
+      "step": 1920
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.699845320607571e-06,
+      "logits/chosen": -1.3387973308563232,
+      "logits/rejected": -0.737518310546875,
+      "logps/chosen": -649.2557373046875,
+      "logps/rejected": -1256.31103515625,
+      "loss": 0.0787,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.2079259604215622,
+      "rewards/margins": 0.2849615216255188,
+      "rewards/rejected": -0.4928874373435974,
+      "step": 1930
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6723414552876052e-06,
+      "logits/chosen": -1.6167535781860352,
+      "logits/rejected": -1.154432773590088,
+      "logps/chosen": -748.0882568359375,
+      "logps/rejected": -1344.591064453125,
+      "loss": 0.0877,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.2506067454814911,
+      "rewards/margins": 0.25507742166519165,
+      "rewards/rejected": -0.5056841969490051,
+      "step": 1940
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6449496416858285e-06,
+      "logits/chosen": -1.4003807306289673,
+      "logits/rejected": -0.9338014721870422,
+      "logps/chosen": -809.3727416992188,
+      "logps/rejected": -1325.3411865234375,
+      "loss": 0.0916,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.31113409996032715,
+      "rewards/margins": 0.2267119586467743,
+      "rewards/rejected": -0.5378460884094238,
+      "step": 1950
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6176735882153284e-06,
+      "logits/chosen": -1.6452484130859375,
+      "logits/rejected": -0.8376502990722656,
+      "logps/chosen": -785.1629028320312,
+      "logps/rejected": -1453.1751708984375,
+      "loss": 0.0648,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.24854576587677002,
+      "rewards/margins": 0.31561970710754395,
+      "rewards/rejected": -0.564165472984314,
+      "step": 1960
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5905169876171223e-06,
+      "logits/chosen": -1.3292005062103271,
+      "logits/rejected": -1.1126399040222168,
+      "logps/chosen": -715.500244140625,
+      "logps/rejected": -1252.599365234375,
+      "loss": 0.0795,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.2204395830631256,
+      "rewards/margins": 0.24764451384544373,
+      "rewards/rejected": -0.46808409690856934,
+      "step": 1970
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.56348351646022e-06,
+      "logits/chosen": -1.7485458850860596,
+      "logits/rejected": -1.010534405708313,
+      "logps/chosen": -690.407958984375,
+      "logps/rejected": -1369.098876953125,
+      "loss": 0.068,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.21612036228179932,
+      "rewards/margins": 0.3184736371040344,
+      "rewards/rejected": -0.534593939781189,
+      "step": 1980
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5365768346438798e-06,
+      "logits/chosen": -1.5913209915161133,
+      "logits/rejected": -0.9691961407661438,
+      "logps/chosen": -642.9549560546875,
+      "logps/rejected": -1401.8271484375,
+      "loss": 0.0482,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.17725911736488342,
+      "rewards/margins": 0.3471994996070862,
+      "rewards/rejected": -0.5244585871696472,
+      "step": 1990
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.509800584902108e-06,
+      "logits/chosen": -1.489768147468567,
+      "logits/rejected": -0.9495828747749329,
+      "logps/chosen": -701.9700317382812,
+      "logps/rejected": -1363.816650390625,
+      "loss": 0.1071,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.21182182431221008,
+      "rewards/margins": 0.28239232301712036,
+      "rewards/rejected": -0.4942142069339752,
+      "step": 2000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4831583923105e-06,
+      "logits/chosen": -1.5869534015655518,
+      "logits/rejected": -0.7756304740905762,
+      "logps/chosen": -702.1505737304688,
+      "logps/rejected": -1240.6885986328125,
+      "loss": 0.0858,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.17973639070987701,
+      "rewards/margins": 0.26760077476501465,
+      "rewards/rejected": -0.44733715057373047,
+      "step": 2010
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4566538637954556e-06,
+      "logits/chosen": -1.632197618484497,
+      "logits/rejected": -1.0302485227584839,
+      "logps/chosen": -545.2433471679688,
+      "logps/rejected": -1228.961669921875,
+      "loss": 0.0726,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.15146581828594208,
+      "rewards/margins": 0.30513542890548706,
+      "rewards/rejected": -0.45660123229026794,
+      "step": 2020
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.430290587645865e-06,
+      "logits/chosen": -1.6993017196655273,
+      "logits/rejected": -1.0094877481460571,
+      "logps/chosen": -642.3917236328125,
+      "logps/rejected": -1230.881103515625,
+      "loss": 0.075,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.17171506583690643,
+      "rewards/margins": 0.2653944194316864,
+      "rewards/rejected": -0.43710947036743164,
+      "step": 2030
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4040721330273063e-06,
+      "logits/chosen": -1.5952852964401245,
+      "logits/rejected": -1.0270891189575195,
+      "logps/chosen": -582.1309814453125,
+      "logps/rejected": -1269.764892578125,
+      "loss": 0.0746,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.1761622130870819,
+      "rewards/margins": 0.2915706932544708,
+      "rewards/rejected": -0.46773290634155273,
+      "step": 2040
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3780020494988447e-06,
+      "logits/chosen": -1.3955180644989014,
+      "logits/rejected": -0.8177375793457031,
+      "logps/chosen": -722.6862182617188,
+      "logps/rejected": -1415.1123046875,
+      "loss": 0.0664,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.24629633128643036,
+      "rewards/margins": 0.30118346214294434,
+      "rewards/rejected": -0.5474798083305359,
+      "step": 2050
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3520838665324704e-06,
+      "logits/chosen": -1.5909086465835571,
+      "logits/rejected": -1.0436404943466187,
+      "logps/chosen": -642.248291015625,
+      "logps/rejected": -1155.293212890625,
+      "loss": 0.1028,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.233381986618042,
+      "rewards/margins": 0.2285003662109375,
+      "rewards/rejected": -0.4618823528289795,
+      "step": 2060
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3263210930352737e-06,
+      "logits/chosen": -1.6532557010650635,
+      "logits/rejected": -1.1368399858474731,
+      "logps/chosen": -737.0447387695312,
+      "logps/rejected": -1343.158203125,
+      "loss": 0.076,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.2329857349395752,
+      "rewards/margins": 0.27646780014038086,
+      "rewards/rejected": -0.5094535946846008,
+      "step": 2070
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3007172168743854e-06,
+      "logits/chosen": -1.4727094173431396,
+      "logits/rejected": -0.934604287147522,
+      "logps/chosen": -640.7996826171875,
+      "logps/rejected": -1427.772216796875,
+      "loss": 0.0472,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.19586428999900818,
+      "rewards/margins": 0.3154672384262085,
+      "rewards/rejected": -0.5113314986228943,
+      "step": 2080
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.275275704404783e-06,
+      "logits/chosen": -1.6787872314453125,
+      "logits/rejected": -1.2128015756607056,
+      "logps/chosen": -691.6954345703125,
+      "logps/rejected": -1384.380859375,
+      "loss": 0.0654,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.19925867021083832,
+      "rewards/margins": 0.3383576273918152,
+      "rewards/rejected": -0.5376163721084595,
+      "step": 2090
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2500000000000007e-06,
+      "logits/chosen": -1.4302475452423096,
+      "logits/rejected": -0.8775968551635742,
+      "logps/chosen": -651.3941040039062,
+      "logps/rejected": -1211.107177734375,
+      "loss": 0.0971,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.20647192001342773,
+      "rewards/margins": 0.2682201862335205,
+      "rewards/rejected": -0.4746921956539154,
+      "step": 2100
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2248935255858118e-06,
+      "logits/chosen": -1.6478124856948853,
+      "logits/rejected": -0.8647529482841492,
+      "logps/chosen": -657.9767456054688,
+      "logps/rejected": -1445.98291015625,
+      "loss": 0.0464,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.1874004602432251,
+      "rewards/margins": 0.3412622809410095,
+      "rewards/rejected": -0.5286628007888794,
+      "step": 2110
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1999596801769617e-06,
+      "logits/chosen": -1.3120667934417725,
+      "logits/rejected": -0.8835352063179016,
+      "logps/chosen": -623.9490356445312,
+      "logps/rejected": -1219.7760009765625,
+      "loss": 0.0814,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.20036788284778595,
+      "rewards/margins": 0.27707821130752563,
+      "rewards/rejected": -0.4774460792541504,
+      "step": 2120
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1752018394169882e-06,
+      "logits/chosen": -1.7081232070922852,
+      "logits/rejected": -0.9557887315750122,
+      "logps/chosen": -739.8932495117188,
+      "logps/rejected": -1354.7886962890625,
+      "loss": 0.063,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1747320294380188,
+      "rewards/margins": 0.3161642551422119,
+      "rewards/rejected": -0.4908962845802307,
+      "step": 2130
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1506233551212186e-06,
+      "logits/chosen": -1.655374526977539,
+      "logits/rejected": -0.7405561208724976,
+      "logps/chosen": -661.6314697265625,
+      "logps/rejected": -1389.5576171875,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.17144644260406494,
+      "rewards/margins": 0.3735453486442566,
+      "rewards/rejected": -0.5449918508529663,
+      "step": 2140
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1262275548229852e-06,
+      "logits/chosen": -1.257212519645691,
+      "logits/rejected": -0.9685500860214233,
+      "logps/chosen": -522.0648193359375,
+      "logps/rejected": -1224.8739013671875,
+      "loss": 0.0756,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.16989552974700928,
+      "rewards/margins": 0.29921796917915344,
+      "rewards/rejected": -0.4691134989261627,
+      "step": 2150
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1020177413231334e-06,
+      "logits/chosen": -1.2228362560272217,
+      "logits/rejected": -0.7848671674728394,
+      "logps/chosen": -745.6207275390625,
+      "logps/rejected": -1389.057373046875,
+      "loss": 0.0749,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.23939558863639832,
+      "rewards/margins": 0.28681597113609314,
+      "rewards/rejected": -0.5262116193771362,
+      "step": 2160
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.077997192242871e-06,
+      "logits/chosen": -1.4660097360610962,
+      "logits/rejected": -0.974969208240509,
+      "logps/chosen": -637.7769165039062,
+      "logps/rejected": -1316.93017578125,
+      "loss": 0.065,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.15726247429847717,
+      "rewards/margins": 0.31098154187202454,
+      "rewards/rejected": -0.4682440757751465,
+      "step": 2170
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0541691595800338e-06,
+      "logits/chosen": -1.7434594631195068,
+      "logits/rejected": -0.9831005930900574,
+      "logps/chosen": -688.5543212890625,
+      "logps/rejected": -1286.400634765625,
+      "loss": 0.0618,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.19961665570735931,
+      "rewards/margins": 0.28607600927352905,
+      "rewards/rejected": -0.48569273948669434,
+      "step": 2180
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0305368692688175e-06,
+      "logits/chosen": -1.7225663661956787,
+      "logits/rejected": -1.016959309577942,
+      "logps/chosen": -655.6326904296875,
+      "logps/rejected": -1282.568115234375,
+      "loss": 0.0711,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.19480985403060913,
+      "rewards/margins": 0.30676570534706116,
+      "rewards/rejected": -0.5015754699707031,
+      "step": 2190
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0071035207430352e-06,
+      "logits/chosen": -1.6399486064910889,
+      "logits/rejected": -1.1826612949371338,
+      "logps/chosen": -634.7337646484375,
+      "logps/rejected": -1298.8203125,
+      "loss": 0.0641,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.1779237538576126,
+      "rewards/margins": 0.31172335147857666,
+      "rewards/rejected": -0.4896470904350281,
+      "step": 2200
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.838722865029674e-07,
+      "logits/chosen": -1.6983495950698853,
+      "logits/rejected": -0.968538761138916,
+      "logps/chosen": -577.1983642578125,
+      "logps/rejected": -1137.4327392578125,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.150478333234787,
+      "rewards/margins": 0.2877087891101837,
+      "rewards/rejected": -0.4381870627403259,
+      "step": 2210
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.608463116858544e-07,
+      "logits/chosen": -1.341863989830017,
+      "logits/rejected": -0.9842678904533386,
+      "logps/chosen": -667.66015625,
+      "logps/rejected": -1293.984130859375,
+      "loss": 0.0759,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.21305403113365173,
+      "rewards/margins": 0.2678312361240387,
+      "rewards/rejected": -0.48088526725769043,
+      "step": 2220
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.380287136401001e-07,
+      "logits/chosen": -1.5788919925689697,
+      "logits/rejected": -0.9648571014404297,
+      "logps/chosen": -630.4158325195312,
+      "logps/rejected": -1360.427001953125,
+      "loss": 0.07,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1882987916469574,
+      "rewards/margins": 0.3067074418067932,
+      "rewards/rejected": -0.495006263256073,
+      "step": 2230
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.154225815032242e-07,
+      "logits/chosen": -1.4030269384384155,
+      "logits/rejected": -0.967923641204834,
+      "logps/chosen": -672.2672119140625,
+      "logps/rejected": -1179.1063232421875,
+      "loss": 0.0872,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.20492562651634216,
+      "rewards/margins": 0.24575969576835632,
+      "rewards/rejected": -0.4506853520870209,
+      "step": 2240
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.930309757836517e-07,
+      "logits/chosen": -1.5156638622283936,
+      "logits/rejected": -1.0182697772979736,
+      "logps/chosen": -763.292724609375,
+      "logps/rejected": -1367.725341796875,
+      "loss": 0.094,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.2593090534210205,
+      "rewards/margins": 0.27271029353141785,
+      "rewards/rejected": -0.532019317150116,
+      "step": 2250
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.708569279463622e-07,
+      "logits/chosen": -1.521528959274292,
+      "logits/rejected": -0.7515259981155396,
+      "logps/chosen": -701.5262451171875,
+      "logps/rejected": -1299.10693359375,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.2057270109653473,
+      "rewards/margins": 0.3127078413963318,
+      "rewards/rejected": -0.5184348821640015,
+      "step": 2260
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.489034400024812e-07,
+      "logits/chosen": -1.5848454236984253,
+      "logits/rejected": -1.0163071155548096,
+      "logps/chosen": -675.6258544921875,
+      "logps/rejected": -1327.443603515625,
+      "loss": 0.0724,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.2220022976398468,
+      "rewards/margins": 0.297424852848053,
+      "rewards/rejected": -0.5194271802902222,
+      "step": 2270
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.271734841028553e-07,
+      "logits/chosen": -1.6320521831512451,
+      "logits/rejected": -0.905319094657898,
+      "logps/chosen": -691.1963500976562,
+      "logps/rejected": -1361.6270751953125,
+      "loss": 0.0634,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1820472925901413,
+      "rewards/margins": 0.337322473526001,
+      "rewards/rejected": -0.5193697810173035,
+      "step": 2280
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.056700021356695e-07,
+      "logits/chosen": -1.7158523797988892,
+      "logits/rejected": -1.1264632940292358,
+      "logps/chosen": -621.7196044921875,
+      "logps/rejected": -1224.8692626953125,
+      "loss": 0.0612,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.17123040556907654,
+      "rewards/margins": 0.28949856758117676,
+      "rewards/rejected": -0.4607289731502533,
+      "step": 2290
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.843959053281663e-07,
+      "logits/chosen": -1.6252124309539795,
+      "logits/rejected": -1.1049168109893799,
+      "logps/chosen": -624.00390625,
+      "logps/rejected": -1279.696044921875,
+      "loss": 0.0827,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.18267154693603516,
+      "rewards/margins": 0.31361207365989685,
+      "rewards/rejected": -0.496283620595932,
+      "step": 2300
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.633540738525066e-07,
+      "logits/chosen": -1.5565481185913086,
+      "logits/rejected": -1.1449733972549438,
+      "logps/chosen": -699.10400390625,
+      "logps/rejected": -1397.216796875,
+      "loss": 0.0733,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.18508002161979675,
+      "rewards/margins": 0.33612412214279175,
+      "rewards/rejected": -0.5212041735649109,
+      "step": 2310
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.425473564358457e-07,
+      "logits/chosen": -1.5639346837997437,
+      "logits/rejected": -1.0611830949783325,
+      "logps/chosen": -636.5604248046875,
+      "logps/rejected": -1440.516357421875,
+      "loss": 0.0714,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.18250158429145813,
+      "rewards/margins": 0.3447728157043457,
+      "rewards/rejected": -0.5272743701934814,
+      "step": 2320
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.219785699746573e-07,
+      "logits/chosen": -1.436949610710144,
+      "logits/rejected": -1.1259959936141968,
+      "logps/chosen": -717.715576171875,
+      "logps/rejected": -1378.193359375,
+      "loss": 0.0757,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.22643479704856873,
+      "rewards/margins": 0.29645177721977234,
+      "rewards/rejected": -0.5228865742683411,
+      "step": 2330
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.016504991533727e-07,
+      "logits/chosen": -1.5969784259796143,
+      "logits/rejected": -0.9250253438949585,
+      "logps/chosen": -714.7103881835938,
+      "logps/rejected": -1392.815185546875,
+      "loss": 0.0606,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.20075106620788574,
+      "rewards/margins": 0.321119487285614,
+      "rewards/rejected": -0.5218705534934998,
+      "step": 2340
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.815658960673782e-07,
+      "logits/chosen": -1.3801023960113525,
+      "logits/rejected": -0.8244778513908386,
+      "logps/chosen": -762.7855834960938,
+      "logps/rejected": -1422.857177734375,
+      "loss": 0.0722,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.22916767001152039,
+      "rewards/margins": 0.2947383224964142,
+      "rewards/rejected": -0.5239059925079346,
+      "step": 2350
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.617274798504286e-07,
+      "logits/chosen": -1.501314640045166,
+      "logits/rejected": -1.1214609146118164,
+      "logps/chosen": -670.3170776367188,
+      "logps/rejected": -1361.3216552734375,
+      "loss": 0.0577,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1791990101337433,
+      "rewards/margins": 0.3246581256389618,
+      "rewards/rejected": -0.5038571357727051,
+      "step": 2360
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.421379363065142e-07,
+      "logits/chosen": -1.3665945529937744,
+      "logits/rejected": -0.8138443231582642,
+      "logps/chosen": -703.3013916015625,
+      "logps/rejected": -1420.8404541015625,
+      "loss": 0.0468,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.20666304230690002,
+      "rewards/margins": 0.3045525550842285,
+      "rewards/rejected": -0.5112155675888062,
+      "step": 2370
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.227999175462521e-07,
+      "logits/chosen": -1.4125187397003174,
+      "logits/rejected": -1.0406558513641357,
+      "logps/chosen": -726.2548828125,
+      "logps/rejected": -1322.2655029296875,
+      "loss": 0.0807,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.1995895951986313,
+      "rewards/margins": 0.27303341031074524,
+      "rewards/rejected": -0.47262296080589294,
+      "step": 2380
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.037160416278279e-07,
+      "logits/chosen": -1.5233482122421265,
+      "logits/rejected": -0.8162555694580078,
+      "logps/chosen": -642.1959228515625,
+      "logps/rejected": -1301.4527587890625,
+      "loss": 0.0589,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.18815341591835022,
+      "rewards/margins": 0.29811039566993713,
+      "rewards/rejected": -0.48626384139060974,
+      "step": 2390
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.848888922025553e-07,
+      "logits/chosen": -1.6081314086914062,
+      "logits/rejected": -1.1109412908554077,
+      "logps/chosen": -642.6141357421875,
+      "logps/rejected": -1297.754150390625,
+      "loss": 0.0596,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.16165341436862946,
+      "rewards/margins": 0.29734665155410767,
+      "rewards/rejected": -0.45900002121925354,
+      "step": 2400
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.663210181650905e-07,
+      "logits/chosen": -1.4959220886230469,
+      "logits/rejected": -1.2034730911254883,
+      "logps/chosen": -575.3321533203125,
+      "logps/rejected": -1197.032470703125,
+      "loss": 0.0979,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.14985838532447815,
+      "rewards/margins": 0.26243895292282104,
+      "rewards/rejected": -0.4122973084449768,
+      "step": 2410
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.48014933308352e-07,
+      "logits/chosen": -1.6470867395401,
+      "logits/rejected": -1.3305919170379639,
+      "logps/chosen": -586.1883544921875,
+      "logps/rejected": -1270.7503662109375,
+      "loss": 0.0629,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.17088322341442108,
+      "rewards/margins": 0.29427218437194824,
+      "rewards/rejected": -0.4651554226875305,
+      "step": 2420
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.299731159831953e-07,
+      "logits/chosen": -1.416419506072998,
+      "logits/rejected": -0.9456356167793274,
+      "logps/chosen": -537.1207275390625,
+      "logps/rejected": -1264.95556640625,
+      "loss": 0.0765,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.15687143802642822,
+      "rewards/margins": 0.3016758859157562,
+      "rewards/rejected": -0.45854735374450684,
+      "step": 2430
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.121980087628802e-07,
+      "logits/chosen": -1.742026686668396,
+      "logits/rejected": -1.1850230693817139,
+      "logps/chosen": -548.540283203125,
+      "logps/rejected": -1200.302734375,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.15248605608940125,
+      "rewards/margins": 0.29664915800094604,
+      "rewards/rejected": -0.4491352438926697,
+      "step": 2440
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.946920181123904e-07,
+      "logits/chosen": -1.5911678075790405,
+      "logits/rejected": -0.8707629442214966,
+      "logps/chosen": -719.4567260742188,
+      "logps/rejected": -1296.5850830078125,
+      "loss": 0.062,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.19431941211223602,
+      "rewards/margins": 0.2867654860019684,
+      "rewards/rejected": -0.4810849130153656,
+      "step": 2450
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.774575140626317e-07,
+      "logits/chosen": -1.5443083047866821,
+      "logits/rejected": -1.0547590255737305,
+      "logps/chosen": -627.6547241210938,
+      "logps/rejected": -1386.137451171875,
+      "loss": 0.0469,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.19703541696071625,
+      "rewards/margins": 0.36265698075294495,
+      "rewards/rejected": -0.5596923828125,
+      "step": 2460
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.6049682988957037e-07,
+      "logits/chosen": -1.7288634777069092,
+      "logits/rejected": -0.8852437138557434,
+      "logps/chosen": -762.8133544921875,
+      "logps/rejected": -1455.493896484375,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.1930183470249176,
+      "rewards/margins": 0.3427968919277191,
+      "rewards/rejected": -0.5358152389526367,
+      "step": 2470
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.438122617983442e-07,
+      "logits/chosen": -1.6805801391601562,
+      "logits/rejected": -1.0256656408309937,
+      "logps/chosen": -660.5001220703125,
+      "logps/rejected": -1275.15625,
+      "loss": 0.0772,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.208724707365036,
+      "rewards/margins": 0.28110271692276,
+      "rewards/rejected": -0.48982739448547363,
+      "step": 2480
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.27406068612396e-07,
+      "logits/chosen": -1.6279792785644531,
+      "logits/rejected": -0.9308198094367981,
+      "logps/chosen": -610.5615844726562,
+      "logps/rejected": -1186.8466796875,
+      "loss": 0.0817,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.15405488014221191,
+      "rewards/margins": 0.27832460403442383,
+      "rewards/rejected": -0.43237948417663574,
+      "step": 2490
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.1128047146765936e-07,
+      "logits/chosen": -1.5780996084213257,
+      "logits/rejected": -0.9356195330619812,
+      "logps/chosen": -742.1123046875,
+      "logps/rejected": -1309.2537841796875,
+      "loss": 0.0887,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.21093162894248962,
+      "rewards/margins": 0.29827579855918884,
+      "rewards/rejected": -0.5092074275016785,
+      "step": 2500
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.954376535118562e-07,
+      "logits/chosen": -1.451724648475647,
+      "logits/rejected": -1.0602259635925293,
+      "logps/chosen": -743.4791870117188,
+      "logps/rejected": -1402.3712158203125,
+      "loss": 0.058,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.21480798721313477,
+      "rewards/margins": 0.3146688640117645,
+      "rewards/rejected": -0.5294768214225769,
+      "step": 2510
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.798797596089351e-07,
+      "logits/chosen": -1.3941175937652588,
+      "logits/rejected": -0.991183876991272,
+      "logps/chosen": -676.042236328125,
+      "logps/rejected": -1340.735595703125,
+      "loss": 0.0826,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.23510794341564178,
+      "rewards/margins": 0.2676921486854553,
+      "rewards/rejected": -0.5028000473976135,
+      "step": 2520
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.646088960486863e-07,
+      "logits/chosen": -1.6253753900527954,
+      "logits/rejected": -1.1532971858978271,
+      "logps/chosen": -519.7340087890625,
+      "logps/rejected": -1152.824462890625,
+      "loss": 0.0921,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.14441636204719543,
+      "rewards/margins": 0.2724858820438385,
+      "rewards/rejected": -0.41690224409103394,
+      "step": 2530
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.4962713026158697e-07,
+      "logits/chosen": -1.607985258102417,
+      "logits/rejected": -1.2400459051132202,
+      "logps/chosen": -500.02020263671875,
+      "logps/rejected": -1161.1961669921875,
+      "loss": 0.0885,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1489984691143036,
+      "rewards/margins": 0.2574361562728882,
+      "rewards/rejected": -0.4064346253871918,
+      "step": 2540
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3493649053890325e-07,
+      "logits/chosen": -1.5089218616485596,
+      "logits/rejected": -0.8405025601387024,
+      "logps/chosen": -708.8694458007812,
+      "logps/rejected": -1470.578857421875,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.21505184471607208,
+      "rewards/margins": 0.34505897760391235,
+      "rewards/rejected": -0.5601108074188232,
+      "step": 2550
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.2053896575809426e-07,
+      "logits/chosen": -1.639923334121704,
+      "logits/rejected": -1.043080449104309,
+      "logps/chosen": -783.112060546875,
+      "logps/rejected": -1343.9681396484375,
+      "loss": 0.0747,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.24453020095825195,
+      "rewards/margins": 0.2654086947441101,
+      "rewards/rejected": -0.5099388957023621,
+      "step": 2560
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.0643650511354483e-07,
+      "logits/chosen": -1.6855356693267822,
+      "logits/rejected": -0.8734441995620728,
+      "logps/chosen": -713.609130859375,
+      "logps/rejected": -1338.822998046875,
+      "loss": 0.0699,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.2088811695575714,
+      "rewards/margins": 0.30321091413497925,
+      "rewards/rejected": -0.512092113494873,
+      "step": 2570
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9263101785268253e-07,
+      "logits/chosen": -1.40034818649292,
+      "logits/rejected": -1.074161171913147,
+      "logps/chosen": -614.1613159179688,
+      "logps/rejected": -1244.793212890625,
+      "loss": 0.0945,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.20661716163158417,
+      "rewards/margins": 0.28118330240249634,
+      "rewards/rejected": -0.4878004193305969,
+      "step": 2580
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.791243730174903e-07,
+      "logits/chosen": -1.6715885400772095,
+      "logits/rejected": -1.1529837846755981,
+      "logps/chosen": -547.1351928710938,
+      "logps/rejected": -1222.099365234375,
+      "loss": 0.0664,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.12953749299049377,
+      "rewards/margins": 0.31725236773490906,
+      "rewards/rejected": -0.44678983092308044,
+      "step": 2590
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.6591839919146963e-07,
+      "logits/chosen": -1.6793832778930664,
+      "logits/rejected": -0.8579391241073608,
+      "logps/chosen": -668.2606201171875,
+      "logps/rejected": -1342.045166015625,
+      "loss": 0.0454,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.16217480599880219,
+      "rewards/margins": 0.35208019614219666,
+      "rewards/rejected": -0.5142549276351929,
+      "step": 2600
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.53014884252083e-07,
+      "logits/chosen": -1.469300627708435,
+      "logits/rejected": -0.767549455165863,
+      "logps/chosen": -720.5267333984375,
+      "logps/rejected": -1461.4560546875,
+      "loss": 0.0383,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.19582341611385345,
+      "rewards/margins": 0.36342793703079224,
+      "rewards/rejected": -0.5592513680458069,
+      "step": 2610
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.404155751286988e-07,
+      "logits/chosen": -1.249018907546997,
+      "logits/rejected": -0.6379393339157104,
+      "logps/chosen": -634.1907958984375,
+      "logps/rejected": -1247.8612060546875,
+      "loss": 0.0579,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.18129587173461914,
+      "rewards/margins": 0.29834333062171936,
+      "rewards/rejected": -0.4796391427516937,
+      "step": 2620
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.281221775660894e-07,
+      "logits/chosen": -1.5495988130569458,
+      "logits/rejected": -0.8539125323295593,
+      "logps/chosen": -634.5662231445312,
+      "logps/rejected": -1174.760498046875,
+      "loss": 0.0756,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.1546512097120285,
+      "rewards/margins": 0.26017943024635315,
+      "rewards/rejected": -0.41483062505722046,
+      "step": 2630
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1613635589349756e-07,
+      "logits/chosen": -1.432281494140625,
+      "logits/rejected": -0.9715501070022583,
+      "logps/chosen": -674.9788818359375,
+      "logps/rejected": -1332.5562744140625,
+      "loss": 0.0969,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.2190193384885788,
+      "rewards/margins": 0.2877424359321594,
+      "rewards/rejected": -0.5067617893218994,
+      "step": 2640
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.044597327993153e-07,
+      "logits/chosen": -1.5616180896759033,
+      "logits/rejected": -1.1563438177108765,
+      "logps/chosen": -606.47509765625,
+      "logps/rejected": -1302.349365234375,
+      "loss": 0.0886,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.17486010491847992,
+      "rewards/margins": 0.30272260308265686,
+      "rewards/rejected": -0.4775826930999756,
+      "step": 2650
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.9309388911139427e-07,
+      "logits/chosen": -1.6554429531097412,
+      "logits/rejected": -0.9854539036750793,
+      "logps/chosen": -705.8251342773438,
+      "logps/rejected": -1167.0521240234375,
+      "loss": 0.0627,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.1600712388753891,
+      "rewards/margins": 0.2684277892112732,
+      "rewards/rejected": -0.4284990429878235,
+      "step": 2660
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.8204036358303173e-07,
+      "logits/chosen": -1.5156413316726685,
+      "logits/rejected": -0.9938446879386902,
+      "logps/chosen": -598.7017822265625,
+      "logps/rejected": -1183.290771484375,
+      "loss": 0.0615,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.16524121165275574,
+      "rewards/margins": 0.28775864839553833,
+      "rewards/rejected": -0.45299988985061646,
+      "step": 2670
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.713006526846439e-07,
+      "logits/chosen": -1.6532678604125977,
+      "logits/rejected": -0.9341715574264526,
+      "logps/chosen": -686.2755126953125,
+      "logps/rejected": -1177.0118408203125,
+      "loss": 0.1065,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.18550768494606018,
+      "rewards/margins": 0.24031631648540497,
+      "rewards/rejected": -0.42582401633262634,
+      "step": 2680
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6087621040117157e-07,
+      "logits/chosen": -1.7682663202285767,
+      "logits/rejected": -0.9737985730171204,
+      "logps/chosen": -617.552001953125,
+      "logps/rejected": -1250.820068359375,
+      "loss": 0.0657,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.16673611104488373,
+      "rewards/margins": 0.3138349950313568,
+      "rewards/rejected": -0.48057112097740173,
+      "step": 2690
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.507684480352292e-07,
+      "logits/chosen": -1.4090583324432373,
+      "logits/rejected": -0.8811914324760437,
+      "logps/chosen": -614.2474975585938,
+      "logps/rejected": -1185.4610595703125,
+      "loss": 0.0782,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.1610531508922577,
+      "rewards/margins": 0.2807305157184601,
+      "rewards/rejected": -0.4417836666107178,
+      "step": 2700
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4097873401604124e-07,
+      "logits/chosen": -1.5380343198776245,
+      "logits/rejected": -1.1294854879379272,
+      "logps/chosen": -611.0988159179688,
+      "logps/rejected": -1329.0465087890625,
+      "loss": 0.0768,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.18254908919334412,
+      "rewards/margins": 0.304229736328125,
+      "rewards/rejected": -0.4867788255214691,
+      "step": 2710
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.31508393714177e-07,
+      "logits/chosen": -1.3804000616073608,
+      "logits/rejected": -0.9093223810195923,
+      "logps/chosen": -625.8529052734375,
+      "logps/rejected": -1367.1817626953125,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.1812657117843628,
+      "rewards/margins": 0.3490126132965088,
+      "rewards/rejected": -0.5302783250808716,
+      "step": 2720
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.223587092621162e-07,
+      "logits/chosen": -1.5867640972137451,
+      "logits/rejected": -0.9399985074996948,
+      "logps/chosen": -676.7730102539062,
+      "logps/rejected": -1228.8385009765625,
+      "loss": 0.0731,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.17374689877033234,
+      "rewards/margins": 0.2911613881587982,
+      "rewards/rejected": -0.46490830183029175,
+      "step": 2730
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1353091938067024e-07,
+      "logits/chosen": -1.3089746236801147,
+      "logits/rejected": -1.0505130290985107,
+      "logps/chosen": -693.4176025390625,
+      "logps/rejected": -1227.46142578125,
+      "loss": 0.108,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.20728778839111328,
+      "rewards/margins": 0.26565903425216675,
+      "rewards/rejected": -0.47294682264328003,
+      "step": 2740
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0502621921127776e-07,
+      "logits/chosen": -1.6415075063705444,
+      "logits/rejected": -1.284096360206604,
+      "logps/chosen": -757.7032470703125,
+      "logps/rejected": -1357.983154296875,
+      "loss": 0.0815,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.2325299233198166,
+      "rewards/margins": 0.27662572264671326,
+      "rewards/rejected": -0.5091556906700134,
+      "step": 2750
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.684576015420277e-08,
+      "logits/chosen": -1.3691219091415405,
+      "logits/rejected": -1.0339184999465942,
+      "logps/chosen": -582.3773193359375,
+      "logps/rejected": -1222.3829345703125,
+      "loss": 0.0626,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.17146027088165283,
+      "rewards/margins": 0.299213707447052,
+      "rewards/rejected": -0.47067388892173767,
+      "step": 2760
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.899064971265275e-08,
+      "logits/chosen": -1.581775188446045,
+      "logits/rejected": -1.0051343441009521,
+      "logps/chosen": -718.9093017578125,
+      "logps/rejected": -1241.8062744140625,
+      "loss": 0.0669,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.1594964563846588,
+      "rewards/margins": 0.28466928005218506,
+      "rewards/rejected": -0.44416576623916626,
+      "step": 2770
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.146195134284052e-08,
+      "logits/chosen": -1.4833767414093018,
+      "logits/rejected": -1.0147238969802856,
+      "logps/chosen": -611.6956176757812,
+      "logps/rejected": -1192.66015625,
+      "loss": 0.0914,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.20624539256095886,
+      "rewards/margins": 0.2417052984237671,
+      "rewards/rejected": -0.4479507505893707,
+      "step": 2780
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.426068431000883e-08,
+      "logits/chosen": -1.4513158798217773,
+      "logits/rejected": -0.7021422982215881,
+      "logps/chosen": -749.6524658203125,
+      "logps/rejected": -1255.1658935546875,
+      "loss": 0.0931,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.2141745388507843,
+      "rewards/margins": 0.2607952654361725,
+      "rewards/rejected": -0.4749697744846344,
+      "step": 2790
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.738782355044048e-08,
+      "logits/chosen": -1.3975991010665894,
+      "logits/rejected": -0.9725745320320129,
+      "logps/chosen": -653.4349365234375,
+      "logps/rejected": -1304.32666015625,
+      "loss": 0.0911,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.2119159698486328,
+      "rewards/margins": 0.2688130736351013,
+      "rewards/rejected": -0.48072901368141174,
+      "step": 2800
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.084429953946452e-08,
+      "logits/chosen": -1.5564731359481812,
+      "logits/rejected": -1.0867516994476318,
+      "logps/chosen": -614.5533447265625,
+      "logps/rejected": -1332.2275390625,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.18066832423210144,
+      "rewards/margins": 0.32241129875183105,
+      "rewards/rejected": -0.5030795335769653,
+      "step": 2810
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.463099816548578e-08,
+      "logits/chosen": -1.6008579730987549,
+      "logits/rejected": -1.274701476097107,
+      "logps/chosen": -693.5843505859375,
+      "logps/rejected": -1352.4996337890625,
+      "loss": 0.0756,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.21343615651130676,
+      "rewards/margins": 0.2871994972229004,
+      "rewards/rejected": -0.5006356835365295,
+      "step": 2820
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.8748760610051735e-08,
+      "logits/chosen": -1.5801304578781128,
+      "logits/rejected": -1.0879913568496704,
+      "logps/chosen": -632.2335205078125,
+      "logps/rejected": -1163.8685302734375,
+      "loss": 0.1023,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.16370609402656555,
+      "rewards/margins": 0.23538489639759064,
+      "rewards/rejected": -0.3990909457206726,
+      "step": 2830
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.319838323396691e-08,
+      "logits/chosen": -1.6419999599456787,
+      "logits/rejected": -1.148740530014038,
+      "logps/chosen": -622.5277099609375,
+      "logps/rejected": -1213.3511962890625,
+      "loss": 0.0825,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.18520578742027283,
+      "rewards/margins": 0.2649041712284088,
+      "rewards/rejected": -0.45010989904403687,
+      "step": 2840
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.798061746947995e-08,
+      "logits/chosen": -1.5798577070236206,
+      "logits/rejected": -1.0320630073547363,
+      "logps/chosen": -704.6697998046875,
+      "logps/rejected": -1352.01806640625,
+      "loss": 0.0633,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.21166665852069855,
+      "rewards/margins": 0.2847224175930023,
+      "rewards/rejected": -0.49638909101486206,
+      "step": 2850
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.309616971855195e-08,
+      "logits/chosen": -1.5923948287963867,
+      "logits/rejected": -0.976751446723938,
+      "logps/chosen": -629.5615844726562,
+      "logps/rejected": -1377.3729248046875,
+      "loss": 0.0713,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.16095896065235138,
+      "rewards/margins": 0.3290019631385803,
+      "rewards/rejected": -0.4899609684944153,
+      "step": 2860
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.8545701257221003e-08,
+      "logits/chosen": -1.5409938097000122,
+      "logits/rejected": -1.0359934568405151,
+      "logps/chosen": -694.2177124023438,
+      "logps/rejected": -1343.0057373046875,
+      "loss": 0.0705,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.20338015258312225,
+      "rewards/margins": 0.2844991981983185,
+      "rewards/rejected": -0.48787933588027954,
+      "step": 2870
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.4329828146074096e-08,
+      "logits/chosen": -1.6312742233276367,
+      "logits/rejected": -1.199237585067749,
+      "logps/chosen": -547.9122924804688,
+      "logps/rejected": -1200.981689453125,
+      "loss": 0.0883,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.14529578387737274,
+      "rewards/margins": 0.2816771864891052,
+      "rewards/rejected": -0.42697301506996155,
+      "step": 2880
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0449121146845773e-08,
+      "logits/chosen": -1.4686208963394165,
+      "logits/rejected": -0.851632297039032,
+      "logps/chosen": -645.3411254882812,
+      "logps/rejected": -1267.0413818359375,
+      "loss": 0.0672,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.20314595103263855,
+      "rewards/margins": 0.28285151720046997,
+      "rewards/rejected": -0.4859974980354309,
+      "step": 2890
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6904105645142443e-08,
+      "logits/chosen": -1.631974220275879,
+      "logits/rejected": -1.05061674118042,
+      "logps/chosen": -530.013671875,
+      "logps/rejected": -1199.45068359375,
+      "loss": 0.0715,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.15192578732967377,
+      "rewards/margins": 0.31625282764434814,
+      "rewards/rejected": -0.4681786596775055,
+      "step": 2900
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3695261579316776e-08,
+      "logits/chosen": -1.4659382104873657,
+      "logits/rejected": -0.9017642736434937,
+      "logps/chosen": -520.8211669921875,
+      "logps/rejected": -1325.6385498046875,
+      "loss": 0.056,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.15825147926807404,
+      "rewards/margins": 0.35485899448394775,
+      "rewards/rejected": -0.5131104588508606,
+      "step": 2910
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.0823023375489128e-08,
+      "logits/chosen": -1.5723521709442139,
+      "logits/rejected": -1.1977678537368774,
+      "logps/chosen": -600.8292236328125,
+      "logps/rejected": -1365.1005859375,
+      "loss": 0.0466,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.16827178001403809,
+      "rewards/margins": 0.3262236714363098,
+      "rewards/rejected": -0.4944954812526703,
+      "step": 2920
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.28777988873486e-09,
+      "logits/chosen": -1.6231420040130615,
+      "logits/rejected": -1.0731710195541382,
+      "logps/chosen": -671.5623779296875,
+      "logps/rejected": -1358.5367431640625,
+      "loss": 0.0688,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.21294662356376648,
+      "rewards/margins": 0.3046295642852783,
+      "rewards/rejected": -0.5175761580467224,
+      "step": 2930
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.089874350439507e-09,
+      "logits/chosen": -1.4259278774261475,
+      "logits/rejected": -0.8798543810844421,
+      "logps/chosen": -694.62548828125,
+      "logps/rejected": -1346.7584228515625,
+      "loss": 0.0846,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.21204762160778046,
+      "rewards/margins": 0.27931392192840576,
+      "rewards/rejected": -0.4913616180419922,
+      "step": 2940
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.229604321829561e-09,
+      "logits/chosen": -1.596842885017395,
+      "logits/rejected": -1.0910255908966064,
+      "logps/chosen": -653.8787231445312,
+      "logps/rejected": -1310.4539794921875,
+      "loss": 0.0672,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.17626503109931946,
+      "rewards/margins": 0.3032272756099701,
+      "rewards/rejected": -0.47949227690696716,
+      "step": 2950
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.7072216536885855e-09,
+      "logits/chosen": -1.3221776485443115,
+      "logits/rejected": -0.8105962872505188,
+      "logps/chosen": -586.7859497070312,
+      "logps/rejected": -1137.4781494140625,
+      "loss": 0.099,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.19391945004463196,
+      "rewards/margins": 0.2519974112510681,
+      "rewards/rejected": -0.4459168314933777,
+      "step": 2960
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.5229324522605949e-09,
+      "logits/chosen": -1.553269624710083,
+      "logits/rejected": -1.0229682922363281,
+      "logps/chosen": -545.6754760742188,
+      "logps/rejected": -1225.473388671875,
+      "loss": 0.0631,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.16567066311836243,
+      "rewards/margins": 0.2939736247062683,
+      "rewards/rejected": -0.45964425802230835,
+      "step": 2970
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.768970513457151e-10,
+      "logits/chosen": -1.6384735107421875,
+      "logits/rejected": -0.9449461102485657,
+      "logps/chosen": -580.2606811523438,
+      "logps/rejected": -1307.0478515625,
+      "loss": 0.0692,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.16747145354747772,
+      "rewards/margins": 0.31812483072280884,
+      "rewards/rejected": -0.48559626936912537,
+      "step": 2980
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.692299905944883e-10,
+      "logits/chosen": -1.3800950050354004,
+      "logits/rejected": -0.8507388234138489,
+      "logps/chosen": -683.5490112304688,
+      "logps/rejected": -1347.037841796875,
+      "loss": 0.052,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.19447694718837738,
+      "rewards/margins": 0.3061479330062866,
+      "rewards/rejected": -0.5006248950958252,
+      "step": 2990
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 0.0,
+      "logits/chosen": -1.6093953847885132,
+      "logits/rejected": -0.8852132558822632,
+      "logps/chosen": -607.3538208007812,
+      "logps/rejected": -1118.9757080078125,
+      "loss": 0.1084,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.13724899291992188,
+      "rewards/margins": 0.2455356866121292,
+      "rewards/rejected": -0.3827846646308899,
+      "step": 3000
+    },
+    {
+      "epoch": 1.0,
+      "step": 3000,
+      "total_flos": 0.0,
+      "train_loss": 0.08193727386991183,
+      "train_runtime": 13050.0697,
+      "train_samples_per_second": 0.92,
+      "train_steps_per_second": 0.23
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}