Training in progress, epoch 1

README.md

@@ -18,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5650
-- Rewards/chosen: 0.0816
-- Rewards/rejected: -0.2564
-- Rewards/accuracies: 0.7695
-- Rewards/margins: 0.3380
-- Logps/rejected: -175.5244
-- Logps/chosen: -271.4002
-- Logits/rejected: -3.0699
-- Logits/chosen: -3.0344
+- Loss: 0.6488
+- Rewards/chosen: 0.0341
+- Rewards/rejected: -0.0820
+- Rewards/accuracies: 0.7109
+- Rewards/margins: 0.1161
+- Logps/rejected: -224.8079
+- Logps/chosen: -271.6428
+- Logits/rejected: -3.0562
+- Logits/chosen: -3.0761
 
 ## Model description
 
@@ -62,12 +62,12 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.482         | 1.0   | 485  | 0.5650          | 0.0816         | -0.2564          | 0.7695             | 0.3380          | -175.5244      | -271.4002    | -3.0699         | -3.0344       |
+| 0.6488        | 1.0   | 485  | 0.6488          | 0.0341         | -0.0820          | 0.7109             | 0.1161          | -224.8079      | -271.6428    | -3.0562         | -3.0761       |
 
 
 ### Framework versions
 
 - Transformers 4.35.0
-- Pytorch 2.1.1+cu121
+- Pytorch 2.1.0+cu121
 - Datasets 2.14.6
 - Tokenizers 0.14.1

adapter_config.json

@@ -16,10 +16,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "o_proj",
     "q_proj",
-    "k_proj"
+    "k_proj",
+    "v_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:74aeaa41361cf19130082481873abd9dee881a558133a02119bf73f216ba81dc
+oid sha256:3f4b73195a43a92625562cb4c520ddd964103582be6f8b732cf7b60aff8d31b2
 size 109086672

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -3.034407377243042,
-    "eval_logits/rejected": -3.069913864135742,
-    "eval_logps/chosen": -271.40020751953125,
-    "eval_logps/rejected": -175.5244140625,
-    "eval_loss": 0.5650191903114319,
-    "eval_rewards/accuracies": 0.76953125,
-    "eval_rewards/chosen": 0.08157022297382355,
-    "eval_rewards/margins": 0.33799096941947937,
-    "eval_rewards/rejected": -0.25642073154449463,
-    "eval_runtime": 254.1285,
+    "eval_logits/chosen": -3.07612943649292,
+    "eval_logits/rejected": -3.056239604949951,
+    "eval_logps/chosen": -271.6427917480469,
+    "eval_logps/rejected": -224.8079376220703,
+    "eval_loss": 0.6488261818885803,
+    "eval_rewards/accuracies": 0.7109375,
+    "eval_rewards/chosen": 0.034067459404468536,
+    "eval_rewards/margins": 0.11610361933708191,
+    "eval_rewards/rejected": -0.08203616738319397,
+    "eval_runtime": 254.1478,
     "eval_samples": 2000,
-    "eval_samples_per_second": 7.87,
+    "eval_samples_per_second": 7.869,
     "eval_steps_per_second": 0.063,
-    "train_loss": 0.5539181610972611,
-    "train_runtime": 15602.6148,
-    "train_samples": 62064,
-    "train_samples_per_second": 3.978,
+    "train_loss": 0.6667533972828659,
+    "train_runtime": 15505.6746,
+    "train_samples": 61966,
+    "train_samples_per_second": 3.996,
     "train_steps_per_second": 0.031
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -3.034407377243042,
-    "eval_logits/rejected": -3.069913864135742,
-    "eval_logps/chosen": -271.40020751953125,
-    "eval_logps/rejected": -175.5244140625,
-    "eval_loss": 0.5650191903114319,
-    "eval_rewards/accuracies": 0.76953125,
-    "eval_rewards/chosen": 0.08157022297382355,
-    "eval_rewards/margins": 0.33799096941947937,
-    "eval_rewards/rejected": -0.25642073154449463,
-    "eval_runtime": 254.1285,
+    "eval_logits/chosen": -3.07612943649292,
+    "eval_logits/rejected": -3.056239604949951,
+    "eval_logps/chosen": -271.6427917480469,
+    "eval_logps/rejected": -224.8079376220703,
+    "eval_loss": 0.6488261818885803,
+    "eval_rewards/accuracies": 0.7109375,
+    "eval_rewards/chosen": 0.034067459404468536,
+    "eval_rewards/margins": 0.11610361933708191,
+    "eval_rewards/rejected": -0.08203616738319397,
+    "eval_runtime": 254.1478,
     "eval_samples": 2000,
-    "eval_samples_per_second": 7.87,
+    "eval_samples_per_second": 7.869,
     "eval_steps_per_second": 0.063
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5539181610972611,
-    "train_runtime": 15602.6148,
-    "train_samples": 62064,
-    "train_samples_per_second": 3.978,
+    "train_loss": 0.6667533972828659,
+    "train_runtime": 15505.6746,
+    "train_samples": 61966,
+    "train_samples_per_second": 3.996,
     "train_steps_per_second": 0.031
 }

trainer_state.json

@@ -11,10 +11,10 @@
     {
       "epoch": 0.0,
       "learning_rate": 1.020408163265306e-08,
-      "logits/chosen": -3.094454526901245,
-      "logits/rejected": -3.0498220920562744,
-      "logps/chosen": -242.99183654785156,
-      "logps/rejected": -74.66817474365234,
+      "logits/chosen": -2.891636610031128,
+      "logits/rejected": -2.8851490020751953,
+      "logps/chosen": -135.91143798828125,
+      "logps/rejected": -101.67433166503906,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,698 +25,698 @@
     {
       "epoch": 0.02,
       "learning_rate": 1.0204081632653061e-07,
-      "logits/chosen": -3.032047986984253,
-      "logits/rejected": -3.029446840286255,
-      "logps/chosen": -290.1824645996094,
-      "logps/rejected": -75.82839965820312,
-      "loss": 0.6935,
-      "rewards/accuracies": 0.4027777910232544,
-      "rewards/chosen": -0.007104851305484772,
-      "rewards/margins": -0.0044839149340987206,
-      "rewards/rejected": -0.0026209354400634766,
+      "logits/chosen": -2.9867801666259766,
+      "logits/rejected": -3.007345199584961,
+      "logps/chosen": -309.9524230957031,
+      "logps/rejected": -272.5204162597656,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": -0.00014034591731615365,
+      "rewards/margins": 0.0023386774118989706,
+      "rewards/rejected": -0.002479023300111294,
       "step": 10
     },
     {
       "epoch": 0.04,
       "learning_rate": 2.0408163265306121e-07,
-      "logits/chosen": -2.9773757457733154,
-      "logits/rejected": -2.967517852783203,
-      "logps/chosen": -297.57342529296875,
-      "logps/rejected": -77.62318420410156,
-      "loss": 0.692,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.00020697650325018913,
-      "rewards/margins": 0.003021990181878209,
-      "rewards/rejected": -0.0028150142170488834,
+      "logits/chosen": -3.0750911235809326,
+      "logits/rejected": -3.0683979988098145,
+      "logps/chosen": -282.82012939453125,
+      "logps/rejected": -249.6508331298828,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.0003494807460810989,
+      "rewards/margins": 0.0025773285888135433,
+      "rewards/rejected": -0.002926809247583151,
       "step": 20
     },
     {
       "epoch": 0.06,
       "learning_rate": 3.0612244897959183e-07,
-      "logits/chosen": -2.983607769012451,
-      "logits/rejected": -2.9363152980804443,
-      "logps/chosen": -288.51458740234375,
-      "logps/rejected": -75.65086364746094,
-      "loss": 0.6892,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.0037677965592592955,
-      "rewards/margins": 0.004846884869039059,
-      "rewards/rejected": -0.008614679798483849,
+      "logits/chosen": -3.072047710418701,
+      "logits/rejected": -3.0354340076446533,
+      "logps/chosen": -280.91827392578125,
+      "logps/rejected": -199.9836883544922,
+      "loss": 0.6943,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.0013565481640398502,
+      "rewards/margins": -0.0006766369333490729,
+      "rewards/rejected": 0.0020331847481429577,
       "step": 30
     },
     {
       "epoch": 0.08,
       "learning_rate": 4.0816326530612243e-07,
-      "logits/chosen": -3.0467514991760254,
-      "logits/rejected": -3.010239362716675,
-      "logps/chosen": -243.7971954345703,
-      "logps/rejected": -81.06056213378906,
-      "loss": 0.685,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.0063628097996115685,
-      "rewards/margins": 0.02118637040257454,
-      "rewards/rejected": -0.014823561534285545,
+      "logits/chosen": -3.0383801460266113,
+      "logits/rejected": -3.035770893096924,
+      "logps/chosen": -290.04510498046875,
+      "logps/rejected": -238.2515106201172,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.0009484182810410857,
+      "rewards/margins": 0.002096892800182104,
+      "rewards/rejected": -0.001148474169895053,
       "step": 40
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.988532110091743e-07,
-      "logits/chosen": -3.0095317363739014,
-      "logits/rejected": -3.0367846488952637,
-      "logps/chosen": -251.5819854736328,
-      "logps/rejected": -78.19547271728516,
-      "loss": 0.6784,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.005416669882833958,
-      "rewards/margins": 0.023932188749313354,
-      "rewards/rejected": -0.018515516072511673,
+      "logits/chosen": -3.037513017654419,
+      "logits/rejected": -3.0122437477111816,
+      "logps/chosen": -296.66009521484375,
+      "logps/rejected": -217.6807861328125,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.006595917046070099,
+      "rewards/margins": 0.010886356234550476,
+      "rewards/rejected": -0.004290440119802952,
       "step": 50
     },
     {
       "epoch": 0.12,
       "learning_rate": 4.873853211009174e-07,
-      "logits/chosen": -3.0116028785705566,
-      "logits/rejected": -3.0300631523132324,
-      "logps/chosen": -281.01361083984375,
-      "logps/rejected": -75.49365997314453,
-      "loss": 0.6715,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.015385298058390617,
-      "rewards/margins": 0.050571341067552567,
-      "rewards/rejected": -0.0351860448718071,
+      "logits/chosen": -3.0581448078155518,
+      "logits/rejected": -3.0058109760284424,
+      "logps/chosen": -297.3258361816406,
+      "logps/rejected": -242.928466796875,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.0021637417376041412,
+      "rewards/margins": 0.0060836682096123695,
+      "rewards/rejected": -0.003919926937669516,
       "step": 60
     },
     {
       "epoch": 0.14,
       "learning_rate": 4.7591743119266054e-07,
-      "logits/chosen": -3.0327250957489014,
-      "logits/rejected": -3.0184121131896973,
-      "logps/chosen": -262.8722229003906,
-      "logps/rejected": -71.65990447998047,
-      "loss": 0.6649,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 0.016824517399072647,
-      "rewards/margins": 0.06025807186961174,
-      "rewards/rejected": -0.043433547019958496,
+      "logits/chosen": -3.0567173957824707,
+      "logits/rejected": -3.056859254837036,
+      "logps/chosen": -281.0410461425781,
+      "logps/rejected": -256.200927734375,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.006569950375705957,
+      "rewards/margins": 0.011877616867423058,
+      "rewards/rejected": -0.005307666026055813,
       "step": 70
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.644495412844037e-07,
-      "logits/chosen": -3.0364532470703125,
-      "logits/rejected": -2.988002300262451,
-      "logps/chosen": -254.49423217773438,
-      "logps/rejected": -70.27412414550781,
-      "loss": 0.6556,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.022701723501086235,
-      "rewards/margins": 0.07623252272605896,
-      "rewards/rejected": -0.05353079363703728,
+      "logits/chosen": -3.0421910285949707,
+      "logits/rejected": -3.0536134243011475,
+      "logps/chosen": -299.5488586425781,
+      "logps/rejected": -233.7873077392578,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.004993592854589224,
+      "rewards/margins": 0.014339953660964966,
+      "rewards/rejected": -0.00934636127203703,
       "step": 80
     },
     {
       "epoch": 0.19,
       "learning_rate": 4.5298165137614677e-07,
-      "logits/chosen": -3.068497657775879,
-      "logits/rejected": -3.0402565002441406,
-      "logps/chosen": -266.61614990234375,
-      "logps/rejected": -81.87393951416016,
-      "loss": 0.6455,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 0.026070792227983475,
-      "rewards/margins": 0.10358123481273651,
-      "rewards/rejected": -0.07751044631004333,
+      "logits/chosen": -3.029874324798584,
+      "logits/rejected": -3.018759250640869,
+      "logps/chosen": -253.44351196289062,
+      "logps/rejected": -201.76646423339844,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.003981114365160465,
+      "rewards/margins": 0.012469857931137085,
+      "rewards/rejected": -0.008488742634654045,
       "step": 90
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.4151376146788986e-07,
-      "logits/chosen": -3.0521655082702637,
-      "logits/rejected": -3.057821750640869,
-      "logps/chosen": -286.0577087402344,
-      "logps/rejected": -77.96414947509766,
-      "loss": 0.6336,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.033475782722234726,
-      "rewards/margins": 0.14013811945915222,
-      "rewards/rejected": -0.10666234791278839,
+      "logits/chosen": -3.0842769145965576,
+      "logits/rejected": -3.086177349090576,
+      "logps/chosen": -258.80426025390625,
+      "logps/rejected": -233.66793823242188,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.004184984136372805,
+      "rewards/margins": 0.017948109656572342,
+      "rewards/rejected": -0.01376312505453825,
       "step": 100
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.30045871559633e-07,
-      "logits/chosen": -3.003532886505127,
-      "logits/rejected": -2.995978355407715,
-      "logps/chosen": -276.5457458496094,
-      "logps/rejected": -80.02079010009766,
-      "loss": 0.6234,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.0331401564180851,
-      "rewards/margins": 0.14480046927928925,
-      "rewards/rejected": -0.11166031658649445,
+      "logits/chosen": -3.050504207611084,
+      "logits/rejected": -3.0392251014709473,
+      "logps/chosen": -275.3699035644531,
+      "logps/rejected": -228.40451049804688,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.007297619245946407,
+      "rewards/margins": 0.02423209697008133,
+      "rewards/rejected": -0.016934476792812347,
       "step": 110
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.1857798165137613e-07,
-      "logits/chosen": -3.0330376625061035,
-      "logits/rejected": -3.030214548110962,
-      "logps/chosen": -276.41632080078125,
-      "logps/rejected": -77.67643737792969,
-      "loss": 0.6164,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.043682295829057693,
-      "rewards/margins": 0.177944153547287,
-      "rewards/rejected": -0.1342618763446808,
+      "logits/chosen": -3.0525004863739014,
+      "logits/rejected": -3.024714946746826,
+      "logps/chosen": -295.898681640625,
+      "logps/rejected": -199.49343872070312,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.007169491611421108,
+      "rewards/margins": 0.02695578895509243,
+      "rewards/rejected": -0.019786298274993896,
       "step": 120
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.071100917431192e-07,
-      "logits/chosen": -2.9754703044891357,
-      "logits/rejected": -2.9898681640625,
-      "logps/chosen": -283.3277587890625,
-      "logps/rejected": -83.87138366699219,
-      "loss": 0.6121,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 0.048630841076374054,
-      "rewards/margins": 0.19439519941806793,
-      "rewards/rejected": -0.14576435089111328,
+      "logits/chosen": -3.026671886444092,
+      "logits/rejected": -3.028925657272339,
+      "logps/chosen": -268.51568603515625,
+      "logps/rejected": -220.26260375976562,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.01651640608906746,
+      "rewards/margins": 0.03892368823289871,
+      "rewards/rejected": -0.022407282143831253,
       "step": 130
     },
     {
       "epoch": 0.29,
       "learning_rate": 3.9564220183486236e-07,
-      "logits/chosen": -3.0477757453918457,
-      "logits/rejected": -3.0237550735473633,
-      "logps/chosen": -291.98065185546875,
-      "logps/rejected": -82.53144073486328,
-      "loss": 0.5997,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.034745730459690094,
-      "rewards/margins": 0.20989501476287842,
-      "rewards/rejected": -0.17514929175376892,
+      "logits/chosen": -3.0486905574798584,
+      "logits/rejected": -3.038287878036499,
+      "logps/chosen": -287.53082275390625,
+      "logps/rejected": -222.7250213623047,
+      "loss": 0.6761,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.015277748927474022,
+      "rewards/margins": 0.03703855723142624,
+      "rewards/rejected": -0.021760808303952217,
       "step": 140
     },
     {
       "epoch": 0.31,
       "learning_rate": 3.841743119266055e-07,
-      "logits/chosen": -3.033001661300659,
-      "logits/rejected": -3.015845775604248,
-      "logps/chosen": -289.15582275390625,
-      "logps/rejected": -76.08447265625,
-      "loss": 0.5925,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 0.0425817035138607,
-      "rewards/margins": 0.21189098060131073,
-      "rewards/rejected": -0.16930925846099854,
+      "logits/chosen": -3.0499072074890137,
+      "logits/rejected": -2.9781885147094727,
+      "logps/chosen": -259.89739990234375,
+      "logps/rejected": -237.8246307373047,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.008082658052444458,
+      "rewards/margins": 0.022740600630640984,
+      "rewards/rejected": -0.014657942578196526,
       "step": 150
     },
     {
       "epoch": 0.33,
       "learning_rate": 3.7270642201834864e-07,
-      "logits/chosen": -3.0720551013946533,
-      "logits/rejected": -3.0518932342529297,
-      "logps/chosen": -271.08258056640625,
-      "logps/rejected": -75.97576141357422,
-      "loss": 0.5874,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 0.03000471368432045,
-      "rewards/margins": 0.20934228599071503,
-      "rewards/rejected": -0.17933759093284607,
+      "logits/chosen": -3.0514883995056152,
+      "logits/rejected": -3.022423505783081,
+      "logps/chosen": -309.00494384765625,
+      "logps/rejected": -228.82583618164062,
+      "loss": 0.6761,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.011194340884685516,
+      "rewards/margins": 0.04575566574931145,
+      "rewards/rejected": -0.03456132486462593,
       "step": 160
     },
     {
       "epoch": 0.35,
       "learning_rate": 3.612385321100918e-07,
-      "logits/chosen": -3.026865243911743,
-      "logits/rejected": -3.030813455581665,
-      "logps/chosen": -287.5133361816406,
-      "logps/rejected": -77.84892272949219,
-      "loss": 0.5811,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.050167638808488846,
-      "rewards/margins": 0.24577708542346954,
-      "rewards/rejected": -0.1956094205379486,
+      "logits/chosen": -3.0084691047668457,
+      "logits/rejected": -3.011247158050537,
+      "logps/chosen": -300.25762939453125,
+      "logps/rejected": -233.0517120361328,
+      "loss": 0.6723,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.017441127449274063,
+      "rewards/margins": 0.04596921056509018,
+      "rewards/rejected": -0.028528084978461266,
       "step": 170
     },
     {
       "epoch": 0.37,
       "learning_rate": 3.497706422018348e-07,
-      "logits/chosen": -3.064037322998047,
-      "logits/rejected": -3.0434131622314453,
-      "logps/chosen": -270.81378173828125,
-      "logps/rejected": -78.64222717285156,
-      "loss": 0.5708,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.0572846345603466,
-      "rewards/margins": 0.27750909328460693,
-      "rewards/rejected": -0.2202244997024536,
+      "logits/chosen": -2.998293161392212,
+      "logits/rejected": -3.03139591217041,
+      "logps/chosen": -299.2662658691406,
+      "logps/rejected": -239.52804565429688,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.02176077291369438,
+      "rewards/margins": 0.04461668059229851,
+      "rewards/rejected": -0.022855903953313828,
       "step": 180
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.3830275229357795e-07,
-      "logits/chosen": -3.0381369590759277,
-      "logits/rejected": -3.031832456588745,
-      "logps/chosen": -273.7306823730469,
-      "logps/rejected": -79.31744384765625,
-      "loss": 0.5604,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": 0.05553610250353813,
-      "rewards/margins": 0.29081013798713684,
-      "rewards/rejected": -0.2352740317583084,
+      "logits/chosen": -3.0845465660095215,
+      "logits/rejected": -3.0398240089416504,
+      "logps/chosen": -292.7340087890625,
+      "logps/rejected": -236.06533813476562,
+      "loss": 0.671,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.026021122932434082,
+      "rewards/margins": 0.05270993709564209,
+      "rewards/rejected": -0.02668880857527256,
       "step": 190
     },
     {
       "epoch": 0.41,
       "learning_rate": 3.268348623853211e-07,
-      "logits/chosen": -3.036811113357544,
-      "logits/rejected": -3.0287680625915527,
-      "logps/chosen": -266.4691467285156,
-      "logps/rejected": -77.38215637207031,
-      "loss": 0.5504,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.08118367195129395,
-      "rewards/margins": 0.3425747752189636,
-      "rewards/rejected": -0.2613911032676697,
+      "logits/chosen": -3.0351808071136475,
+      "logits/rejected": -3.048321485519409,
+      "logps/chosen": -269.8604736328125,
+      "logps/rejected": -221.87197875976562,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.023518767207860947,
+      "rewards/margins": 0.06251207739114761,
+      "rewards/rejected": -0.03899329900741577,
       "step": 200
     },
     {
       "epoch": 0.43,
       "learning_rate": 3.1536697247706423e-07,
-      "logits/chosen": -3.061699867248535,
-      "logits/rejected": -3.042888641357422,
-      "logps/chosen": -269.961181640625,
-      "logps/rejected": -89.21647644042969,
-      "loss": 0.5501,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 0.07142322510480881,
-      "rewards/margins": 0.3240587115287781,
-      "rewards/rejected": -0.25263547897338867,
+      "logits/chosen": -3.0768158435821533,
+      "logits/rejected": -3.083721160888672,
+      "logps/chosen": -282.7914733886719,
+      "logps/rejected": -258.88677978515625,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.017539020627737045,
+      "rewards/margins": 0.06356575340032578,
+      "rewards/rejected": -0.04602673649787903,
       "step": 210
     },
     {
       "epoch": 0.45,
       "learning_rate": 3.038990825688073e-07,
-      "logits/chosen": -3.04771089553833,
-      "logits/rejected": -3.018721103668213,
-      "logps/chosen": -250.44091796875,
-      "logps/rejected": -72.33317565917969,
-      "loss": 0.5488,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.06637217104434967,
-      "rewards/margins": 0.3276647627353668,
-      "rewards/rejected": -0.26129260659217834,
+      "logits/chosen": -3.031602621078491,
+      "logits/rejected": -3.0251471996307373,
+      "logps/chosen": -291.6885681152344,
+      "logps/rejected": -229.2044219970703,
+      "loss": 0.6661,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.019487539306282997,
+      "rewards/margins": 0.06684577465057373,
+      "rewards/rejected": -0.047358229756355286,
       "step": 220
     },
     {
       "epoch": 0.47,
       "learning_rate": 2.9243119266055045e-07,
-      "logits/chosen": -2.9626972675323486,
-      "logits/rejected": -2.9827158451080322,
-      "logps/chosen": -293.9212646484375,
-      "logps/rejected": -72.2821044921875,
-      "loss": 0.5313,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": 0.08349540829658508,
-      "rewards/margins": 0.3892216682434082,
-      "rewards/rejected": -0.30572623014450073,
+      "logits/chosen": -3.0594446659088135,
+      "logits/rejected": -3.0538389682769775,
+      "logps/chosen": -296.71978759765625,
+      "logps/rejected": -232.9663543701172,
+      "loss": 0.6672,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.012631967663764954,
+      "rewards/margins": 0.05379491299390793,
+      "rewards/rejected": -0.04116294905543327,
       "step": 230
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.809633027522936e-07,
-      "logits/chosen": -3.034790277481079,
-      "logits/rejected": -3.016634225845337,
-      "logps/chosen": -280.6105651855469,
-      "logps/rejected": -76.09197235107422,
-      "loss": 0.5333,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.08378176391124725,
-      "rewards/margins": 0.4068339467048645,
-      "rewards/rejected": -0.32305219769477844,
+      "logits/chosen": -3.009617805480957,
+      "logits/rejected": -3.0026957988739014,
+      "logps/chosen": -244.1639862060547,
+      "logps/rejected": -207.7158203125,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.01613594964146614,
+      "rewards/margins": 0.0652671605348587,
+      "rewards/rejected": -0.049131207168102264,
       "step": 240
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.6949541284403673e-07,
-      "logits/chosen": -3.0789849758148193,
-      "logits/rejected": -3.0785841941833496,
-      "logps/chosen": -264.5536804199219,
-      "logps/rejected": -82.22047424316406,
-      "loss": 0.5282,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.06328760087490082,
-      "rewards/margins": 0.40200409293174744,
-      "rewards/rejected": -0.3387165069580078,
+      "logits/chosen": -3.0107674598693848,
+      "logits/rejected": -3.012376308441162,
+      "logps/chosen": -287.5134582519531,
+      "logps/rejected": -248.5124053955078,
+      "loss": 0.6593,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.017785798758268356,
+      "rewards/margins": 0.0654246062040329,
+      "rewards/rejected": -0.04763881862163544,
       "step": 250
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.5802752293577976e-07,
-      "logits/chosen": -2.9741625785827637,
-      "logits/rejected": -2.9866743087768555,
-      "logps/chosen": -282.30902099609375,
-      "logps/rejected": -70.76858520507812,
-      "loss": 0.5277,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 0.10191468149423599,
-      "rewards/margins": 0.39590951800346375,
-      "rewards/rejected": -0.29399481415748596,
+      "logits/chosen": -3.014228105545044,
+      "logits/rejected": -2.980214834213257,
+      "logps/chosen": -297.2572021484375,
+      "logps/rejected": -246.603515625,
+      "loss": 0.6592,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.016676222905516624,
+      "rewards/margins": 0.07184126228094101,
+      "rewards/rejected": -0.055165041238069534,
       "step": 260
     },
     {
       "epoch": 0.56,
       "learning_rate": 2.465596330275229e-07,
-      "logits/chosen": -3.032557964324951,
-      "logits/rejected": -3.03240704536438,
-      "logps/chosen": -274.0851135253906,
-      "logps/rejected": -86.98384094238281,
-      "loss": 0.5135,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.07479412853717804,
-      "rewards/margins": 0.4109489321708679,
-      "rewards/rejected": -0.3361548185348511,
+      "logits/chosen": -3.016359329223633,
+      "logits/rejected": -3.0183348655700684,
+      "logps/chosen": -261.7985534667969,
+      "logps/rejected": -230.5518341064453,
+      "loss": 0.6631,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.03012824058532715,
+      "rewards/margins": 0.07465063035488129,
+      "rewards/rejected": -0.04452239349484444,
       "step": 270
     },
     {
       "epoch": 0.58,
       "learning_rate": 2.3509174311926604e-07,
-      "logits/chosen": -3.060285806655884,
-      "logits/rejected": -2.9775302410125732,
-      "logps/chosen": -253.785888671875,
-      "logps/rejected": -70.39444732666016,
-      "loss": 0.5183,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.07235217839479446,
-      "rewards/margins": 0.3860532343387604,
-      "rewards/rejected": -0.31370100378990173,
+      "logits/chosen": -3.062016487121582,
+      "logits/rejected": -3.08595871925354,
+      "logps/chosen": -271.8788757324219,
+      "logps/rejected": -242.447509765625,
+      "loss": 0.6615,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03031134605407715,
+      "rewards/margins": 0.06919924914836884,
+      "rewards/rejected": -0.03888789564371109,
       "step": 280
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.2362385321100916e-07,
-      "logits/chosen": -3.029343843460083,
-      "logits/rejected": -3.0406129360198975,
-      "logps/chosen": -276.57196044921875,
-      "logps/rejected": -84.54597473144531,
-      "loss": 0.5107,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 0.08857797086238861,
-      "rewards/margins": 0.4803849756717682,
-      "rewards/rejected": -0.3918069899082184,
+      "logits/chosen": -3.065378427505493,
+      "logits/rejected": -3.067957639694214,
+      "logps/chosen": -293.88592529296875,
+      "logps/rejected": -247.717529296875,
+      "loss": 0.6595,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.018432429060339928,
+      "rewards/margins": 0.06363337486982346,
+      "rewards/rejected": -0.04520093649625778,
       "step": 290
     },
     {
       "epoch": 0.62,
       "learning_rate": 2.121559633027523e-07,
-      "logits/chosen": -2.9938578605651855,
-      "logits/rejected": -2.9954426288604736,
-      "logps/chosen": -273.7822265625,
-      "logps/rejected": -77.98421478271484,
-      "loss": 0.5079,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 0.08799968659877777,
-      "rewards/margins": 0.40502768754959106,
-      "rewards/rejected": -0.3170279860496521,
+      "logits/chosen": -3.0399653911590576,
+      "logits/rejected": -3.050255298614502,
+      "logps/chosen": -248.15798950195312,
+      "logps/rejected": -231.6765594482422,
+      "loss": 0.6583,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.011154914274811745,
+      "rewards/margins": 0.07162971049547195,
+      "rewards/rejected": -0.060474805533885956,
       "step": 300
     },
     {
       "epoch": 0.64,
       "learning_rate": 2.0068807339449538e-07,
-      "logits/chosen": -3.052614212036133,
-      "logits/rejected": -3.0461201667785645,
-      "logps/chosen": -281.28814697265625,
-      "logps/rejected": -81.84606170654297,
-      "loss": 0.5038,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 0.05326849967241287,
-      "rewards/margins": 0.46244749426841736,
-      "rewards/rejected": -0.4091789722442627,
+      "logits/chosen": -3.0448567867279053,
+      "logits/rejected": -3.0284125804901123,
+      "logps/chosen": -259.03173828125,
+      "logps/rejected": -213.7626190185547,
+      "loss": 0.6551,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.03144986182451248,
+      "rewards/margins": 0.09296337515115738,
+      "rewards/rejected": -0.061513520777225494,
       "step": 310
     },
     {
       "epoch": 0.66,
       "learning_rate": 1.8922018348623852e-07,
-      "logits/chosen": -3.031501054763794,
-      "logits/rejected": -3.042961597442627,
-      "logps/chosen": -271.274658203125,
-      "logps/rejected": -87.3827133178711,
-      "loss": 0.5003,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.07084844261407852,
-      "rewards/margins": 0.445441871881485,
-      "rewards/rejected": -0.37459343671798706,
+      "logits/chosen": -3.069314479827881,
+      "logits/rejected": -3.0522797107696533,
+      "logps/chosen": -247.6428680419922,
+      "logps/rejected": -224.86416625976562,
+      "loss": 0.6537,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.03007657267153263,
+      "rewards/margins": 0.09640363603830338,
+      "rewards/rejected": -0.0663270577788353,
       "step": 320
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.7775229357798163e-07,
-      "logits/chosen": -3.0476019382476807,
-      "logits/rejected": -3.0447893142700195,
-      "logps/chosen": -249.735595703125,
-      "logps/rejected": -73.10395812988281,
-      "loss": 0.4976,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.06198754906654358,
-      "rewards/margins": 0.43834322690963745,
-      "rewards/rejected": -0.37635567784309387,
+      "logits/chosen": -3.0534980297088623,
+      "logits/rejected": -3.0750725269317627,
+      "logps/chosen": -292.9278564453125,
+      "logps/rejected": -239.49560546875,
+      "loss": 0.654,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.03384874761104584,
+      "rewards/margins": 0.09248127043247223,
+      "rewards/rejected": -0.05863253027200699,
       "step": 330
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.6628440366972477e-07,
-      "logits/chosen": -3.055901288986206,
-      "logits/rejected": -3.0517029762268066,
-      "logps/chosen": -273.3477478027344,
-      "logps/rejected": -85.53290557861328,
-      "loss": 0.496,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.08338963240385056,
-      "rewards/margins": 0.5042273998260498,
-      "rewards/rejected": -0.42083778977394104,
+      "logits/chosen": -3.0116381645202637,
+      "logits/rejected": -3.012748956680298,
+      "logps/chosen": -310.517822265625,
+      "logps/rejected": -256.17578125,
+      "loss": 0.6583,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.023617586120963097,
+      "rewards/margins": 0.07958104461431503,
+      "rewards/rejected": -0.055963464081287384,
       "step": 340
     },
     {
       "epoch": 0.72,
       "learning_rate": 1.5481651376146786e-07,
-      "logits/chosen": -3.063744306564331,
-      "logits/rejected": -3.066366195678711,
-      "logps/chosen": -277.1488952636719,
-      "logps/rejected": -88.2572250366211,
-      "loss": 0.4931,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.07289155572652817,
-      "rewards/margins": 0.5126849412918091,
-      "rewards/rejected": -0.4397934079170227,
+      "logits/chosen": -3.015288829803467,
+      "logits/rejected": -3.035534381866455,
+      "logps/chosen": -238.67788696289062,
+      "logps/rejected": -216.6863250732422,
+      "loss": 0.6575,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.016075262799859047,
+      "rewards/margins": 0.07468070089817047,
+      "rewards/rejected": -0.05860542505979538,
       "step": 350
     },
     {
       "epoch": 0.74,
       "learning_rate": 1.43348623853211e-07,
-      "logits/chosen": -3.0237436294555664,
-      "logits/rejected": -3.0258359909057617,
-      "logps/chosen": -292.0096740722656,
-      "logps/rejected": -81.93167114257812,
-      "loss": 0.4951,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": 0.07367613166570663,
-      "rewards/margins": 0.49797001481056213,
-      "rewards/rejected": -0.4242939352989197,
+      "logits/chosen": -2.999647855758667,
+      "logits/rejected": -2.999812602996826,
+      "logps/chosen": -260.34814453125,
+      "logps/rejected": -228.0465545654297,
+      "loss": 0.6576,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.010297578759491444,
+      "rewards/margins": 0.07942849397659302,
+      "rewards/rejected": -0.06913091242313385,
       "step": 360
     },
     {
       "epoch": 0.76,
       "learning_rate": 1.318807339449541e-07,
-      "logits/chosen": -2.9882092475891113,
-      "logits/rejected": -2.9637956619262695,
-      "logps/chosen": -274.551513671875,
-      "logps/rejected": -73.8973388671875,
-      "loss": 0.496,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.0880483016371727,
-      "rewards/margins": 0.49274787306785583,
-      "rewards/rejected": -0.4046996533870697,
+      "logits/chosen": -3.029534101486206,
+      "logits/rejected": -3.0314173698425293,
+      "logps/chosen": -284.08721923828125,
+      "logps/rejected": -248.7538604736328,
+      "loss": 0.6532,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.036643363535404205,
+      "rewards/margins": 0.10196901857852936,
+      "rewards/rejected": -0.06532564014196396,
       "step": 370
     },
     {
       "epoch": 0.78,
       "learning_rate": 1.2041284403669725e-07,
-      "logits/chosen": -3.070621967315674,
-      "logits/rejected": -3.0683789253234863,
-      "logps/chosen": -266.607177734375,
-      "logps/rejected": -81.02775573730469,
-      "loss": 0.493,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.10891600698232651,
-      "rewards/margins": 0.5303564071655273,
-      "rewards/rejected": -0.42144036293029785,
+      "logits/chosen": -3.0199809074401855,
+      "logits/rejected": -3.012413501739502,
+      "logps/chosen": -237.28573608398438,
+      "logps/rejected": -243.95590209960938,
+      "loss": 0.6538,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.012126882560551167,
+      "rewards/margins": 0.08107715100049973,
+      "rewards/rejected": -0.06895027309656143,
       "step": 380
     },
     {
       "epoch": 0.8,
       "learning_rate": 1.0894495412844036e-07,
-      "logits/chosen": -3.0497114658355713,
-      "logits/rejected": -3.053192615509033,
-      "logps/chosen": -280.43218994140625,
-      "logps/rejected": -80.42735290527344,
-      "loss": 0.4892,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.10893626511096954,
-      "rewards/margins": 0.5605167746543884,
-      "rewards/rejected": -0.4515805244445801,
+      "logits/chosen": -2.9979848861694336,
+      "logits/rejected": -3.015382766723633,
+      "logps/chosen": -312.633544921875,
+      "logps/rejected": -235.29806518554688,
+      "loss": 0.6503,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.013257297687232494,
+      "rewards/margins": 0.09175875037908554,
+      "rewards/rejected": -0.07850147038698196,
       "step": 390
     },
     {
       "epoch": 0.82,
       "learning_rate": 9.747706422018348e-08,
-      "logits/chosen": -3.002933979034424,
-      "logits/rejected": -3.0063657760620117,
-      "logps/chosen": -241.24276733398438,
-      "logps/rejected": -75.92924499511719,
-      "loss": 0.4833,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 0.07781459391117096,
-      "rewards/margins": 0.46425342559814453,
-      "rewards/rejected": -0.38643890619277954,
+      "logits/chosen": -3.048086166381836,
+      "logits/rejected": -3.0504872798919678,
+      "logps/chosen": -278.25189208984375,
+      "logps/rejected": -248.26510620117188,
+      "loss": 0.6511,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.01932488940656185,
+      "rewards/margins": 0.07586248964071274,
+      "rewards/rejected": -0.05653759837150574,
       "step": 400
     },
     {
       "epoch": 0.85,
       "learning_rate": 8.60091743119266e-08,
-      "logits/chosen": -3.0454163551330566,
-      "logits/rejected": -3.035583972930908,
-      "logps/chosen": -264.18585205078125,
-      "logps/rejected": -78.031982421875,
-      "loss": 0.4744,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.09802711009979248,
-      "rewards/margins": 0.5436574816703796,
-      "rewards/rejected": -0.44563040137290955,
+      "logits/chosen": -3.0442748069763184,
+      "logits/rejected": -3.0469086170196533,
+      "logps/chosen": -291.3175048828125,
+      "logps/rejected": -228.79153442382812,
+      "loss": 0.6492,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.026980062946677208,
+      "rewards/margins": 0.10874257236719131,
+      "rewards/rejected": -0.08176250755786896,
       "step": 410
     },
     {
       "epoch": 0.87,
       "learning_rate": 7.454128440366971e-08,
-      "logits/chosen": -3.0196666717529297,
-      "logits/rejected": -3.0026302337646484,
-      "logps/chosen": -272.02630615234375,
-      "logps/rejected": -82.01240539550781,
-      "loss": 0.481,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 0.08279488980770111,
-      "rewards/margins": 0.5704164505004883,
-      "rewards/rejected": -0.48762160539627075,
+      "logits/chosen": -3.0467171669006348,
+      "logits/rejected": -3.0507471561431885,
+      "logps/chosen": -274.8234558105469,
+      "logps/rejected": -227.38638305664062,
+      "loss": 0.6509,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.021679330617189407,
+      "rewards/margins": 0.10154237598180771,
+      "rewards/rejected": -0.0798630565404892,
       "step": 420
     },
     {
       "epoch": 0.89,
       "learning_rate": 6.307339449541284e-08,
-      "logits/chosen": -3.0509345531463623,
-      "logits/rejected": -3.0137345790863037,
-      "logps/chosen": -262.2018127441406,
-      "logps/rejected": -77.63418579101562,
-      "loss": 0.4731,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.1073322519659996,
-      "rewards/margins": 0.5776056051254272,
-      "rewards/rejected": -0.4702734053134918,
+      "logits/chosen": -3.0327906608581543,
+      "logits/rejected": -3.0403802394866943,
+      "logps/chosen": -256.6832580566406,
+      "logps/rejected": -259.84295654296875,
+      "loss": 0.651,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.014505205675959587,
+      "rewards/margins": 0.08562619239091873,
+      "rewards/rejected": -0.07112099230289459,
       "step": 430
     },
     {
       "epoch": 0.91,
       "learning_rate": 5.1605504587155966e-08,
-      "logits/chosen": -3.0285000801086426,
-      "logits/rejected": -3.0236475467681885,
-      "logps/chosen": -266.83599853515625,
-      "logps/rejected": -77.38362121582031,
-      "loss": 0.476,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 0.08291526138782501,
-      "rewards/margins": 0.4984784722328186,
-      "rewards/rejected": -0.41556310653686523,
+      "logits/chosen": -3.0034422874450684,
+      "logits/rejected": -3.013791799545288,
+      "logps/chosen": -301.6542053222656,
+      "logps/rejected": -234.38381958007812,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.01893182098865509,
+      "rewards/margins": 0.12277624756097794,
+      "rewards/rejected": -0.10384440422058105,
       "step": 440
     },
     {
       "epoch": 0.93,
       "learning_rate": 4.0137614678899086e-08,
-      "logits/chosen": -3.02640438079834,
-      "logits/rejected": -3.011373996734619,
-      "logps/chosen": -295.5868835449219,
-      "logps/rejected": -80.76414489746094,
-      "loss": 0.4707,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": 0.09663239866495132,
-      "rewards/margins": 0.5815601944923401,
-      "rewards/rejected": -0.48492780327796936,
+      "logits/chosen": -3.0094974040985107,
+      "logits/rejected": -2.9743194580078125,
+      "logps/chosen": -290.7796630859375,
+      "logps/rejected": -238.8968963623047,
+      "loss": 0.6521,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.029885241761803627,
+      "rewards/margins": 0.11550422757863998,
+      "rewards/rejected": -0.08561898022890091,
       "step": 450
     },
     {
       "epoch": 0.95,
       "learning_rate": 2.86697247706422e-08,
-      "logits/chosen": -3.0195059776306152,
-      "logits/rejected": -2.988323926925659,
-      "logps/chosen": -300.5026550292969,
-      "logps/rejected": -86.79838562011719,
-      "loss": 0.4808,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": 0.11054690927267075,
-      "rewards/margins": 0.5899176001548767,
-      "rewards/rejected": -0.47937074303627014,
+      "logits/chosen": -3.043740749359131,
+      "logits/rejected": -3.035067081451416,
+      "logps/chosen": -258.144287109375,
+      "logps/rejected": -234.55081176757812,
+      "loss": 0.6483,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.01103687472641468,
+      "rewards/margins": 0.0873931497335434,
+      "rewards/rejected": -0.07635627686977386,
       "step": 460
     },
     {
       "epoch": 0.97,
       "learning_rate": 1.720183486238532e-08,
-      "logits/chosen": -3.0426931381225586,
-      "logits/rejected": -3.0394179821014404,
-      "logps/chosen": -235.52706909179688,
-      "logps/rejected": -73.9857406616211,
-      "loss": 0.4819,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 0.08785500377416611,
-      "rewards/margins": 0.5274263620376587,
-      "rewards/rejected": -0.4395713806152344,
+      "logits/chosen": -3.0199711322784424,
+      "logits/rejected": -3.026895761489868,
+      "logps/chosen": -288.91119384765625,
+      "logps/rejected": -242.3592071533203,
+      "loss": 0.6511,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.03277132660150528,
+      "rewards/margins": 0.10831280797719955,
+      "rewards/rejected": -0.07554147392511368,
       "step": 470
     },
     {
       "epoch": 0.99,
       "learning_rate": 5.73394495412844e-09,
-      "logits/chosen": -3.0092616081237793,
-      "logits/rejected": -2.972731590270996,
-      "logps/chosen": -249.88876342773438,
-      "logps/rejected": -85.80451965332031,
-      "loss": 0.482,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": 0.07512323558330536,
-      "rewards/margins": 0.5230099558830261,
-      "rewards/rejected": -0.44788676500320435,
+      "logits/chosen": -3.041350841522217,
+      "logits/rejected": -3.058216094970703,
+      "logps/chosen": -258.9853820800781,
+      "logps/rejected": -226.7718048095703,
+      "loss": 0.6488,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.022982869297266006,
+      "rewards/margins": 0.09263849258422852,
+      "rewards/rejected": -0.06965561956167221,
       "step": 480
     },
     {
       "epoch": 1.0,
-      "eval_logits/chosen": -3.034407377243042,
-      "eval_logits/rejected": -3.069913864135742,
-      "eval_logps/chosen": -271.40020751953125,
-      "eval_logps/rejected": -175.5244140625,
-      "eval_loss": 0.5650191903114319,
-      "eval_rewards/accuracies": 0.76953125,
-      "eval_rewards/chosen": 0.08157022297382355,
-      "eval_rewards/margins": 0.33799096941947937,
-      "eval_rewards/rejected": -0.25642073154449463,
-      "eval_runtime": 256.4523,
-      "eval_samples_per_second": 7.799,
-      "eval_steps_per_second": 0.062,
+      "eval_logits/chosen": -3.07612943649292,
+      "eval_logits/rejected": -3.056239604949951,
+      "eval_logps/chosen": -271.6427917480469,
+      "eval_logps/rejected": -224.8079376220703,
+      "eval_loss": 0.6488261818885803,
+      "eval_rewards/accuracies": 0.7109375,
+      "eval_rewards/chosen": 0.034067459404468536,
+      "eval_rewards/margins": 0.11610361933708191,
+      "eval_rewards/rejected": -0.08203616738319397,
+      "eval_runtime": 255.1726,
+      "eval_samples_per_second": 7.838,
+      "eval_steps_per_second": 0.063,
       "step": 485
     },
     {
       "epoch": 1.0,
       "step": 485,
       "total_flos": 0.0,
-      "train_loss": 0.5539181610972611,
-      "train_runtime": 15602.6148,
-      "train_samples_per_second": 3.978,
+      "train_loss": 0.6667533972828659,
+      "train_runtime": 15505.6746,
+      "train_samples_per_second": 3.996,
       "train_steps_per_second": 0.031
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:178ca2e9659218d5d6a040bc9b839def7f1e2d04d37ea8a694c883bf79442126
+oid sha256:c0e4c37c4ba74a7c42b124e93bdaa61e543cd3533851bf4e87301d3ef2e466cd
 size 5688