Training in progress, epoch 2

README.md

@@ -18,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6488
-- Rewards/chosen: 0.0341
-- Rewards/rejected: -0.0820
-- Rewards/accuracies: 0.7109
-- Rewards/margins: 0.1161
-- Logps/rejected: -224.8079
-- Logps/chosen: -271.6428
-- Logits/rejected: -3.0562
-- Logits/chosen: -3.0761
+- Loss: 0.5650
+- Rewards/chosen: 0.0816
+- Rewards/rejected: -0.2564
+- Rewards/accuracies: 0.7695
+- Rewards/margins: 0.3380
+- Logps/rejected: -175.5244
+- Logps/chosen: -271.4002
+- Logits/rejected: -3.0699
+- Logits/chosen: -3.0344
 
 ## Model description
 
@@ -62,12 +62,12 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6488        | 1.0   | 485  | 0.6488          | 0.0341         | -0.0820          | 0.7109             | 0.1161          | -224.8079      | -271.6428    | -3.0562         | -3.0761       |
+| 0.482         | 1.0   | 485  | 0.5650          | 0.0816         | -0.2564          | 0.7695             | 0.3380          | -175.5244      | -271.4002    | -3.0699         | -3.0344       |
 
 
 ### Framework versions
 
 - Transformers 4.35.0
-- Pytorch 2.1.0+cu121
+- Pytorch 2.1.1+cu121
 - Datasets 2.14.6
 - Tokenizers 0.14.1

adapter_config.json

@@ -16,10 +16,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
-    "k_proj",
     "v_proj",
-    "o_proj"
+    "o_proj",
+    "q_proj",
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66d77ec56d43708a60162692263571a3ce844cbf97a6b876e051e1f68a3c50a6
+oid sha256:d078e6f25bcc0098b398028c055fd9e8c6049ebcaaf725b429870480ca5b84c7
 size 109086672

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -3.07612943649292,
-    "eval_logits/rejected": -3.056239604949951,
-    "eval_logps/chosen": -271.6427917480469,
-    "eval_logps/rejected": -224.8079376220703,
-    "eval_loss": 0.6488261818885803,
-    "eval_rewards/accuracies": 0.7109375,
-    "eval_rewards/chosen": 0.034067459404468536,
-    "eval_rewards/margins": 0.11610361933708191,
-    "eval_rewards/rejected": -0.08203616738319397,
-    "eval_runtime": 254.1478,
+    "eval_logits/chosen": -3.034407377243042,
+    "eval_logits/rejected": -3.069913864135742,
+    "eval_logps/chosen": -271.40020751953125,
+    "eval_logps/rejected": -175.5244140625,
+    "eval_loss": 0.5650191903114319,
+    "eval_rewards/accuracies": 0.76953125,
+    "eval_rewards/chosen": 0.08157022297382355,
+    "eval_rewards/margins": 0.33799096941947937,
+    "eval_rewards/rejected": -0.25642073154449463,
+    "eval_runtime": 254.1285,
     "eval_samples": 2000,
-    "eval_samples_per_second": 7.869,
+    "eval_samples_per_second": 7.87,
     "eval_steps_per_second": 0.063,
-    "train_loss": 0.6667533972828659,
-    "train_runtime": 15505.6746,
-    "train_samples": 61966,
-    "train_samples_per_second": 3.996,
+    "train_loss": 0.5539181610972611,
+    "train_runtime": 15602.6148,
+    "train_samples": 62064,
+    "train_samples_per_second": 3.978,
     "train_steps_per_second": 0.031
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -3.07612943649292,
-    "eval_logits/rejected": -3.056239604949951,
-    "eval_logps/chosen": -271.6427917480469,
-    "eval_logps/rejected": -224.8079376220703,
-    "eval_loss": 0.6488261818885803,
-    "eval_rewards/accuracies": 0.7109375,
-    "eval_rewards/chosen": 0.034067459404468536,
-    "eval_rewards/margins": 0.11610361933708191,
-    "eval_rewards/rejected": -0.08203616738319397,
-    "eval_runtime": 254.1478,
+    "eval_logits/chosen": -3.034407377243042,
+    "eval_logits/rejected": -3.069913864135742,
+    "eval_logps/chosen": -271.40020751953125,
+    "eval_logps/rejected": -175.5244140625,
+    "eval_loss": 0.5650191903114319,
+    "eval_rewards/accuracies": 0.76953125,
+    "eval_rewards/chosen": 0.08157022297382355,
+    "eval_rewards/margins": 0.33799096941947937,
+    "eval_rewards/rejected": -0.25642073154449463,
+    "eval_runtime": 254.1285,
     "eval_samples": 2000,
-    "eval_samples_per_second": 7.869,
+    "eval_samples_per_second": 7.87,
     "eval_steps_per_second": 0.063
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6667533972828659,
-    "train_runtime": 15505.6746,
-    "train_samples": 61966,
-    "train_samples_per_second": 3.996,
+    "train_loss": 0.5539181610972611,
+    "train_runtime": 15602.6148,
+    "train_samples": 62064,
+    "train_samples_per_second": 3.978,
     "train_steps_per_second": 0.031
 }

trainer_state.json

@@ -11,10 +11,10 @@
     {
       "epoch": 0.0,
       "learning_rate": 1.020408163265306e-08,
-      "logits/chosen": -2.891636610031128,
-      "logits/rejected": -2.8851490020751953,
-      "logps/chosen": -135.91143798828125,
-      "logps/rejected": -101.67433166503906,
+      "logits/chosen": -3.094454526901245,
+      "logits/rejected": -3.0498220920562744,
+      "logps/chosen": -242.99183654785156,
+      "logps/rejected": -74.66817474365234,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,698 +25,698 @@
     {
       "epoch": 0.02,
       "learning_rate": 1.0204081632653061e-07,
-      "logits/chosen": -2.9867801666259766,
-      "logits/rejected": -3.007345199584961,
-      "logps/chosen": -309.9524230957031,
-      "logps/rejected": -272.5204162597656,
-      "loss": 0.6926,
-      "rewards/accuracies": 0.4444444477558136,
-      "rewards/chosen": -0.00014034591731615365,
-      "rewards/margins": 0.0023386774118989706,
-      "rewards/rejected": -0.002479023300111294,
+      "logits/chosen": -3.032047986984253,
+      "logits/rejected": -3.029446840286255,
+      "logps/chosen": -290.1824645996094,
+      "logps/rejected": -75.82839965820312,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.4027777910232544,
+      "rewards/chosen": -0.007104851305484772,
+      "rewards/margins": -0.0044839149340987206,
+      "rewards/rejected": -0.0026209354400634766,
       "step": 10
     },
     {
       "epoch": 0.04,
       "learning_rate": 2.0408163265306121e-07,
-      "logits/chosen": -3.0750911235809326,
-      "logits/rejected": -3.0683979988098145,
-      "logps/chosen": -282.82012939453125,
-      "logps/rejected": -249.6508331298828,
-      "loss": 0.6922,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -0.0003494807460810989,
-      "rewards/margins": 0.0025773285888135433,
-      "rewards/rejected": -0.002926809247583151,
+      "logits/chosen": -2.9773757457733154,
+      "logits/rejected": -2.967517852783203,
+      "logps/chosen": -297.57342529296875,
+      "logps/rejected": -77.62318420410156,
+      "loss": 0.692,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.00020697650325018913,
+      "rewards/margins": 0.003021990181878209,
+      "rewards/rejected": -0.0028150142170488834,
       "step": 20
     },
     {
       "epoch": 0.06,
       "learning_rate": 3.0612244897959183e-07,
-      "logits/chosen": -3.072047710418701,
-      "logits/rejected": -3.0354340076446533,
-      "logps/chosen": -280.91827392578125,
-      "logps/rejected": -199.9836883544922,
-      "loss": 0.6943,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 0.0013565481640398502,
-      "rewards/margins": -0.0006766369333490729,
-      "rewards/rejected": 0.0020331847481429577,
+      "logits/chosen": -2.983607769012451,
+      "logits/rejected": -2.9363152980804443,
+      "logps/chosen": -288.51458740234375,
+      "logps/rejected": -75.65086364746094,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.0037677965592592955,
+      "rewards/margins": 0.004846884869039059,
+      "rewards/rejected": -0.008614679798483849,
       "step": 30
     },
     {
       "epoch": 0.08,
       "learning_rate": 4.0816326530612243e-07,
-      "logits/chosen": -3.0383801460266113,
-      "logits/rejected": -3.035770893096924,
-      "logps/chosen": -290.04510498046875,
-      "logps/rejected": -238.2515106201172,
-      "loss": 0.6927,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.0009484182810410857,
-      "rewards/margins": 0.002096892800182104,
-      "rewards/rejected": -0.001148474169895053,
+      "logits/chosen": -3.0467514991760254,
+      "logits/rejected": -3.010239362716675,
+      "logps/chosen": -243.7971954345703,
+      "logps/rejected": -81.06056213378906,
+      "loss": 0.685,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0063628097996115685,
+      "rewards/margins": 0.02118637040257454,
+      "rewards/rejected": -0.014823561534285545,
       "step": 40
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.988532110091743e-07,
-      "logits/chosen": -3.037513017654419,
-      "logits/rejected": -3.0122437477111816,
-      "logps/chosen": -296.66009521484375,
-      "logps/rejected": -217.6807861328125,
-      "loss": 0.6896,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.006595917046070099,
-      "rewards/margins": 0.010886356234550476,
-      "rewards/rejected": -0.004290440119802952,
+      "logits/chosen": -3.0095317363739014,
+      "logits/rejected": -3.0367846488952637,
+      "logps/chosen": -251.5819854736328,
+      "logps/rejected": -78.19547271728516,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.005416669882833958,
+      "rewards/margins": 0.023932188749313354,
+      "rewards/rejected": -0.018515516072511673,
       "step": 50
     },
     {
       "epoch": 0.12,
       "learning_rate": 4.873853211009174e-07,
-      "logits/chosen": -3.0581448078155518,
-      "logits/rejected": -3.0058109760284424,
-      "logps/chosen": -297.3258361816406,
-      "logps/rejected": -242.928466796875,
-      "loss": 0.6895,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": 0.0021637417376041412,
-      "rewards/margins": 0.0060836682096123695,
-      "rewards/rejected": -0.003919926937669516,
+      "logits/chosen": -3.0116028785705566,
+      "logits/rejected": -3.0300631523132324,
+      "logps/chosen": -281.01361083984375,
+      "logps/rejected": -75.49365997314453,
+      "loss": 0.6715,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.015385298058390617,
+      "rewards/margins": 0.050571341067552567,
+      "rewards/rejected": -0.0351860448718071,
       "step": 60
     },
     {
       "epoch": 0.14,
       "learning_rate": 4.7591743119266054e-07,
-      "logits/chosen": -3.0567173957824707,
-      "logits/rejected": -3.056859254837036,
-      "logps/chosen": -281.0410461425781,
-      "logps/rejected": -256.200927734375,
-      "loss": 0.6886,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.006569950375705957,
-      "rewards/margins": 0.011877616867423058,
-      "rewards/rejected": -0.005307666026055813,
+      "logits/chosen": -3.0327250957489014,
+      "logits/rejected": -3.0184121131896973,
+      "logps/chosen": -262.8722229003906,
+      "logps/rejected": -71.65990447998047,
+      "loss": 0.6649,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 0.016824517399072647,
+      "rewards/margins": 0.06025807186961174,
+      "rewards/rejected": -0.043433547019958496,
       "step": 70
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.644495412844037e-07,
-      "logits/chosen": -3.0421910285949707,
-      "logits/rejected": -3.0536134243011475,
-      "logps/chosen": -299.5488586425781,
-      "logps/rejected": -233.7873077392578,
-      "loss": 0.6871,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 0.004993592854589224,
-      "rewards/margins": 0.014339953660964966,
-      "rewards/rejected": -0.00934636127203703,
+      "logits/chosen": -3.0364532470703125,
+      "logits/rejected": -2.988002300262451,
+      "logps/chosen": -254.49423217773438,
+      "logps/rejected": -70.27412414550781,
+      "loss": 0.6556,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.022701723501086235,
+      "rewards/margins": 0.07623252272605896,
+      "rewards/rejected": -0.05353079363703728,
       "step": 80
     },
     {
       "epoch": 0.19,
       "learning_rate": 4.5298165137614677e-07,
-      "logits/chosen": -3.029874324798584,
-      "logits/rejected": -3.018759250640869,
-      "logps/chosen": -253.44351196289062,
-      "logps/rejected": -201.76646423339844,
-      "loss": 0.6854,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 0.003981114365160465,
-      "rewards/margins": 0.012469857931137085,
-      "rewards/rejected": -0.008488742634654045,
+      "logits/chosen": -3.068497657775879,
+      "logits/rejected": -3.0402565002441406,
+      "logps/chosen": -266.61614990234375,
+      "logps/rejected": -81.87393951416016,
+      "loss": 0.6455,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 0.026070792227983475,
+      "rewards/margins": 0.10358123481273651,
+      "rewards/rejected": -0.07751044631004333,
       "step": 90
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.4151376146788986e-07,
-      "logits/chosen": -3.0842769145965576,
-      "logits/rejected": -3.086177349090576,
-      "logps/chosen": -258.80426025390625,
-      "logps/rejected": -233.66793823242188,
-      "loss": 0.6863,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.004184984136372805,
-      "rewards/margins": 0.017948109656572342,
-      "rewards/rejected": -0.01376312505453825,
+      "logits/chosen": -3.0521655082702637,
+      "logits/rejected": -3.057821750640869,
+      "logps/chosen": -286.0577087402344,
+      "logps/rejected": -77.96414947509766,
+      "loss": 0.6336,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.033475782722234726,
+      "rewards/margins": 0.14013811945915222,
+      "rewards/rejected": -0.10666234791278839,
       "step": 100
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.30045871559633e-07,
-      "logits/chosen": -3.050504207611084,
-      "logits/rejected": -3.0392251014709473,
-      "logps/chosen": -275.3699035644531,
-      "logps/rejected": -228.40451049804688,
-      "loss": 0.6809,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.007297619245946407,
-      "rewards/margins": 0.02423209697008133,
-      "rewards/rejected": -0.016934476792812347,
+      "logits/chosen": -3.003532886505127,
+      "logits/rejected": -2.995978355407715,
+      "logps/chosen": -276.5457458496094,
+      "logps/rejected": -80.02079010009766,
+      "loss": 0.6234,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.0331401564180851,
+      "rewards/margins": 0.14480046927928925,
+      "rewards/rejected": -0.11166031658649445,
       "step": 110
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.1857798165137613e-07,
-      "logits/chosen": -3.0525004863739014,
-      "logits/rejected": -3.024714946746826,
-      "logps/chosen": -295.898681640625,
-      "logps/rejected": -199.49343872070312,
-      "loss": 0.6804,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.007169491611421108,
-      "rewards/margins": 0.02695578895509243,
-      "rewards/rejected": -0.019786298274993896,
+      "logits/chosen": -3.0330376625061035,
+      "logits/rejected": -3.030214548110962,
+      "logps/chosen": -276.41632080078125,
+      "logps/rejected": -77.67643737792969,
+      "loss": 0.6164,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.043682295829057693,
+      "rewards/margins": 0.177944153547287,
+      "rewards/rejected": -0.1342618763446808,
       "step": 120
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.071100917431192e-07,
-      "logits/chosen": -3.026671886444092,
-      "logits/rejected": -3.028925657272339,
-      "logps/chosen": -268.51568603515625,
-      "logps/rejected": -220.26260375976562,
-      "loss": 0.6796,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.01651640608906746,
-      "rewards/margins": 0.03892368823289871,
-      "rewards/rejected": -0.022407282143831253,
+      "logits/chosen": -2.9754703044891357,
+      "logits/rejected": -2.9898681640625,
+      "logps/chosen": -283.3277587890625,
+      "logps/rejected": -83.87138366699219,
+      "loss": 0.6121,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 0.048630841076374054,
+      "rewards/margins": 0.19439519941806793,
+      "rewards/rejected": -0.14576435089111328,
       "step": 130
     },
     {
       "epoch": 0.29,
       "learning_rate": 3.9564220183486236e-07,
-      "logits/chosen": -3.0486905574798584,
-      "logits/rejected": -3.038287878036499,
-      "logps/chosen": -287.53082275390625,
-      "logps/rejected": -222.7250213623047,
-      "loss": 0.6761,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.015277748927474022,
-      "rewards/margins": 0.03703855723142624,
-      "rewards/rejected": -0.021760808303952217,
+      "logits/chosen": -3.0477757453918457,
+      "logits/rejected": -3.0237550735473633,
+      "logps/chosen": -291.98065185546875,
+      "logps/rejected": -82.53144073486328,
+      "loss": 0.5997,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.034745730459690094,
+      "rewards/margins": 0.20989501476287842,
+      "rewards/rejected": -0.17514929175376892,
       "step": 140
     },
     {
       "epoch": 0.31,
       "learning_rate": 3.841743119266055e-07,
-      "logits/chosen": -3.0499072074890137,
-      "logits/rejected": -2.9781885147094727,
-      "logps/chosen": -259.89739990234375,
-      "logps/rejected": -237.8246307373047,
-      "loss": 0.6766,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 0.008082658052444458,
-      "rewards/margins": 0.022740600630640984,
-      "rewards/rejected": -0.014657942578196526,
+      "logits/chosen": -3.033001661300659,
+      "logits/rejected": -3.015845775604248,
+      "logps/chosen": -289.15582275390625,
+      "logps/rejected": -76.08447265625,
+      "loss": 0.5925,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 0.0425817035138607,
+      "rewards/margins": 0.21189098060131073,
+      "rewards/rejected": -0.16930925846099854,
       "step": 150
     },
     {
       "epoch": 0.33,
       "learning_rate": 3.7270642201834864e-07,
-      "logits/chosen": -3.0514883995056152,
-      "logits/rejected": -3.022423505783081,
-      "logps/chosen": -309.00494384765625,
-      "logps/rejected": -228.82583618164062,
-      "loss": 0.6761,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.011194340884685516,
-      "rewards/margins": 0.04575566574931145,
-      "rewards/rejected": -0.03456132486462593,
+      "logits/chosen": -3.0720551013946533,
+      "logits/rejected": -3.0518932342529297,
+      "logps/chosen": -271.08258056640625,
+      "logps/rejected": -75.97576141357422,
+      "loss": 0.5874,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 0.03000471368432045,
+      "rewards/margins": 0.20934228599071503,
+      "rewards/rejected": -0.17933759093284607,
       "step": 160
     },
     {
       "epoch": 0.35,
       "learning_rate": 3.612385321100918e-07,
-      "logits/chosen": -3.0084691047668457,
-      "logits/rejected": -3.011247158050537,
-      "logps/chosen": -300.25762939453125,
-      "logps/rejected": -233.0517120361328,
-      "loss": 0.6723,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.017441127449274063,
-      "rewards/margins": 0.04596921056509018,
-      "rewards/rejected": -0.028528084978461266,
+      "logits/chosen": -3.026865243911743,
+      "logits/rejected": -3.030813455581665,
+      "logps/chosen": -287.5133361816406,
+      "logps/rejected": -77.84892272949219,
+      "loss": 0.5811,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.050167638808488846,
+      "rewards/margins": 0.24577708542346954,
+      "rewards/rejected": -0.1956094205379486,
       "step": 170
     },
     {
       "epoch": 0.37,
       "learning_rate": 3.497706422018348e-07,
-      "logits/chosen": -2.998293161392212,
-      "logits/rejected": -3.03139591217041,
-      "logps/chosen": -299.2662658691406,
-      "logps/rejected": -239.52804565429688,
-      "loss": 0.6677,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.02176077291369438,
-      "rewards/margins": 0.04461668059229851,
-      "rewards/rejected": -0.022855903953313828,
+      "logits/chosen": -3.064037322998047,
+      "logits/rejected": -3.0434131622314453,
+      "logps/chosen": -270.81378173828125,
+      "logps/rejected": -78.64222717285156,
+      "loss": 0.5708,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.0572846345603466,
+      "rewards/margins": 0.27750909328460693,
+      "rewards/rejected": -0.2202244997024536,
       "step": 180
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.3830275229357795e-07,
-      "logits/chosen": -3.0845465660095215,
-      "logits/rejected": -3.0398240089416504,
-      "logps/chosen": -292.7340087890625,
-      "logps/rejected": -236.06533813476562,
-      "loss": 0.671,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.026021122932434082,
-      "rewards/margins": 0.05270993709564209,
-      "rewards/rejected": -0.02668880857527256,
+      "logits/chosen": -3.0381369590759277,
+      "logits/rejected": -3.031832456588745,
+      "logps/chosen": -273.7306823730469,
+      "logps/rejected": -79.31744384765625,
+      "loss": 0.5604,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.05553610250353813,
+      "rewards/margins": 0.29081013798713684,
+      "rewards/rejected": -0.2352740317583084,
       "step": 190
     },
     {
       "epoch": 0.41,
       "learning_rate": 3.268348623853211e-07,
-      "logits/chosen": -3.0351808071136475,
-      "logits/rejected": -3.048321485519409,
-      "logps/chosen": -269.8604736328125,
-      "logps/rejected": -221.87197875976562,
-      "loss": 0.6699,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.023518767207860947,
-      "rewards/margins": 0.06251207739114761,
-      "rewards/rejected": -0.03899329900741577,
+      "logits/chosen": -3.036811113357544,
+      "logits/rejected": -3.0287680625915527,
+      "logps/chosen": -266.4691467285156,
+      "logps/rejected": -77.38215637207031,
+      "loss": 0.5504,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.08118367195129395,
+      "rewards/margins": 0.3425747752189636,
+      "rewards/rejected": -0.2613911032676697,
       "step": 200
     },
     {
       "epoch": 0.43,
       "learning_rate": 3.1536697247706423e-07,
-      "logits/chosen": -3.0768158435821533,
-      "logits/rejected": -3.083721160888672,
-      "logps/chosen": -282.7914733886719,
-      "logps/rejected": -258.88677978515625,
-      "loss": 0.6694,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.017539020627737045,
-      "rewards/margins": 0.06356575340032578,
-      "rewards/rejected": -0.04602673649787903,
+      "logits/chosen": -3.061699867248535,
+      "logits/rejected": -3.042888641357422,
+      "logps/chosen": -269.961181640625,
+      "logps/rejected": -89.21647644042969,
+      "loss": 0.5501,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 0.07142322510480881,
+      "rewards/margins": 0.3240587115287781,
+      "rewards/rejected": -0.25263547897338867,
       "step": 210
     },
     {
       "epoch": 0.45,
       "learning_rate": 3.038990825688073e-07,
-      "logits/chosen": -3.031602621078491,
-      "logits/rejected": -3.0251471996307373,
-      "logps/chosen": -291.6885681152344,
-      "logps/rejected": -229.2044219970703,
-      "loss": 0.6661,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.019487539306282997,
-      "rewards/margins": 0.06684577465057373,
-      "rewards/rejected": -0.047358229756355286,
+      "logits/chosen": -3.04771089553833,
+      "logits/rejected": -3.018721103668213,
+      "logps/chosen": -250.44091796875,
+      "logps/rejected": -72.33317565917969,
+      "loss": 0.5488,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.06637217104434967,
+      "rewards/margins": 0.3276647627353668,
+      "rewards/rejected": -0.26129260659217834,
       "step": 220
     },
     {
       "epoch": 0.47,
       "learning_rate": 2.9243119266055045e-07,
-      "logits/chosen": -3.0594446659088135,
-      "logits/rejected": -3.0538389682769775,
-      "logps/chosen": -296.71978759765625,
-      "logps/rejected": -232.9663543701172,
-      "loss": 0.6672,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.012631967663764954,
-      "rewards/margins": 0.05379491299390793,
-      "rewards/rejected": -0.04116294905543327,
+      "logits/chosen": -2.9626972675323486,
+      "logits/rejected": -2.9827158451080322,
+      "logps/chosen": -293.9212646484375,
+      "logps/rejected": -72.2821044921875,
+      "loss": 0.5313,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": 0.08349540829658508,
+      "rewards/margins": 0.3892216682434082,
+      "rewards/rejected": -0.30572623014450073,
       "step": 230
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.809633027522936e-07,
-      "logits/chosen": -3.009617805480957,
-      "logits/rejected": -3.0026957988739014,
-      "logps/chosen": -244.1639862060547,
-      "logps/rejected": -207.7158203125,
-      "loss": 0.6633,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.01613594964146614,
-      "rewards/margins": 0.0652671605348587,
-      "rewards/rejected": -0.049131207168102264,
+      "logits/chosen": -3.034790277481079,
+      "logits/rejected": -3.016634225845337,
+      "logps/chosen": -280.6105651855469,
+      "logps/rejected": -76.09197235107422,
+      "loss": 0.5333,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08378176391124725,
+      "rewards/margins": 0.4068339467048645,
+      "rewards/rejected": -0.32305219769477844,
       "step": 240
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.6949541284403673e-07,
-      "logits/chosen": -3.0107674598693848,
-      "logits/rejected": -3.012376308441162,
-      "logps/chosen": -287.5134582519531,
-      "logps/rejected": -248.5124053955078,
-      "loss": 0.6593,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.017785798758268356,
-      "rewards/margins": 0.0654246062040329,
-      "rewards/rejected": -0.04763881862163544,
+      "logits/chosen": -3.0789849758148193,
+      "logits/rejected": -3.0785841941833496,
+      "logps/chosen": -264.5536804199219,
+      "logps/rejected": -82.22047424316406,
+      "loss": 0.5282,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.06328760087490082,
+      "rewards/margins": 0.40200409293174744,
+      "rewards/rejected": -0.3387165069580078,
       "step": 250
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.5802752293577976e-07,
-      "logits/chosen": -3.014228105545044,
-      "logits/rejected": -2.980214834213257,
-      "logps/chosen": -297.2572021484375,
-      "logps/rejected": -246.603515625,
-      "loss": 0.6592,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.016676222905516624,
-      "rewards/margins": 0.07184126228094101,
-      "rewards/rejected": -0.055165041238069534,
+      "logits/chosen": -2.9741625785827637,
+      "logits/rejected": -2.9866743087768555,
+      "logps/chosen": -282.30902099609375,
+      "logps/rejected": -70.76858520507812,
+      "loss": 0.5277,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 0.10191468149423599,
+      "rewards/margins": 0.39590951800346375,
+      "rewards/rejected": -0.29399481415748596,
       "step": 260
     },
     {
       "epoch": 0.56,
       "learning_rate": 2.465596330275229e-07,
-      "logits/chosen": -3.016359329223633,
-      "logits/rejected": -3.0183348655700684,
-      "logps/chosen": -261.7985534667969,
-      "logps/rejected": -230.5518341064453,
-      "loss": 0.6631,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.03012824058532715,
-      "rewards/margins": 0.07465063035488129,
-      "rewards/rejected": -0.04452239349484444,
+      "logits/chosen": -3.032557964324951,
+      "logits/rejected": -3.03240704536438,
+      "logps/chosen": -274.0851135253906,
+      "logps/rejected": -86.98384094238281,
+      "loss": 0.5135,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.07479412853717804,
+      "rewards/margins": 0.4109489321708679,
+      "rewards/rejected": -0.3361548185348511,
       "step": 270
     },
     {
       "epoch": 0.58,
       "learning_rate": 2.3509174311926604e-07,
-      "logits/chosen": -3.062016487121582,
-      "logits/rejected": -3.08595871925354,
-      "logps/chosen": -271.8788757324219,
-      "logps/rejected": -242.447509765625,
-      "loss": 0.6615,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.03031134605407715,
-      "rewards/margins": 0.06919924914836884,
-      "rewards/rejected": -0.03888789564371109,
+      "logits/chosen": -3.060285806655884,
+      "logits/rejected": -2.9775302410125732,
+      "logps/chosen": -253.785888671875,
+      "logps/rejected": -70.39444732666016,
+      "loss": 0.5183,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.07235217839479446,
+      "rewards/margins": 0.3860532343387604,
+      "rewards/rejected": -0.31370100378990173,
       "step": 280
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.2362385321100916e-07,
-      "logits/chosen": -3.065378427505493,
-      "logits/rejected": -3.067957639694214,
-      "logps/chosen": -293.88592529296875,
-      "logps/rejected": -247.717529296875,
-      "loss": 0.6595,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.018432429060339928,
-      "rewards/margins": 0.06363337486982346,
-      "rewards/rejected": -0.04520093649625778,
+      "logits/chosen": -3.029343843460083,
+      "logits/rejected": -3.0406129360198975,
+      "logps/chosen": -276.57196044921875,
+      "logps/rejected": -84.54597473144531,
+      "loss": 0.5107,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 0.08857797086238861,
+      "rewards/margins": 0.4803849756717682,
+      "rewards/rejected": -0.3918069899082184,
       "step": 290
     },
     {
       "epoch": 0.62,
       "learning_rate": 2.121559633027523e-07,
-      "logits/chosen": -3.0399653911590576,
-      "logits/rejected": -3.050255298614502,
-      "logps/chosen": -248.15798950195312,
-      "logps/rejected": -231.6765594482422,
-      "loss": 0.6583,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.011154914274811745,
-      "rewards/margins": 0.07162971049547195,
-      "rewards/rejected": -0.060474805533885956,
+      "logits/chosen": -2.9938578605651855,
+      "logits/rejected": -2.9954426288604736,
+      "logps/chosen": -273.7822265625,
+      "logps/rejected": -77.98421478271484,
+      "loss": 0.5079,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 0.08799968659877777,
+      "rewards/margins": 0.40502768754959106,
+      "rewards/rejected": -0.3170279860496521,
       "step": 300
     },
     {
       "epoch": 0.64,
       "learning_rate": 2.0068807339449538e-07,
-      "logits/chosen": -3.0448567867279053,
-      "logits/rejected": -3.0284125804901123,
-      "logps/chosen": -259.03173828125,
-      "logps/rejected": -213.7626190185547,
-      "loss": 0.6551,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.03144986182451248,
-      "rewards/margins": 0.09296337515115738,
-      "rewards/rejected": -0.061513520777225494,
+      "logits/chosen": -3.052614212036133,
+      "logits/rejected": -3.0461201667785645,
+      "logps/chosen": -281.28814697265625,
+      "logps/rejected": -81.84606170654297,
+      "loss": 0.5038,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 0.05326849967241287,
+      "rewards/margins": 0.46244749426841736,
+      "rewards/rejected": -0.4091789722442627,
       "step": 310
     },
     {
       "epoch": 0.66,
       "learning_rate": 1.8922018348623852e-07,
-      "logits/chosen": -3.069314479827881,
-      "logits/rejected": -3.0522797107696533,
-      "logps/chosen": -247.6428680419922,
-      "logps/rejected": -224.86416625976562,
-      "loss": 0.6537,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.03007657267153263,
-      "rewards/margins": 0.09640363603830338,
-      "rewards/rejected": -0.0663270577788353,
+      "logits/chosen": -3.031501054763794,
+      "logits/rejected": -3.042961597442627,
+      "logps/chosen": -271.274658203125,
+      "logps/rejected": -87.3827133178711,
+      "loss": 0.5003,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.07084844261407852,
+      "rewards/margins": 0.445441871881485,
+      "rewards/rejected": -0.37459343671798706,
       "step": 320
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.7775229357798163e-07,
-      "logits/chosen": -3.0534980297088623,
-      "logits/rejected": -3.0750725269317627,
-      "logps/chosen": -292.9278564453125,
-      "logps/rejected": -239.49560546875,
-      "loss": 0.654,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.03384874761104584,
-      "rewards/margins": 0.09248127043247223,
-      "rewards/rejected": -0.05863253027200699,
+      "logits/chosen": -3.0476019382476807,
+      "logits/rejected": -3.0447893142700195,
+      "logps/chosen": -249.735595703125,
+      "logps/rejected": -73.10395812988281,
+      "loss": 0.4976,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.06198754906654358,
+      "rewards/margins": 0.43834322690963745,
+      "rewards/rejected": -0.37635567784309387,
       "step": 330
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.6628440366972477e-07,
-      "logits/chosen": -3.0116381645202637,
-      "logits/rejected": -3.012748956680298,
-      "logps/chosen": -310.517822265625,
-      "logps/rejected": -256.17578125,
-      "loss": 0.6583,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.023617586120963097,
-      "rewards/margins": 0.07958104461431503,
-      "rewards/rejected": -0.055963464081287384,
+      "logits/chosen": -3.055901288986206,
+      "logits/rejected": -3.0517029762268066,
+      "logps/chosen": -273.3477478027344,
+      "logps/rejected": -85.53290557861328,
+      "loss": 0.496,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.08338963240385056,
+      "rewards/margins": 0.5042273998260498,
+      "rewards/rejected": -0.42083778977394104,
       "step": 340
     },
     {
       "epoch": 0.72,
       "learning_rate": 1.5481651376146786e-07,
-      "logits/chosen": -3.015288829803467,
-      "logits/rejected": -3.035534381866455,
-      "logps/chosen": -238.67788696289062,
-      "logps/rejected": -216.6863250732422,
-      "loss": 0.6575,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.016075262799859047,
-      "rewards/margins": 0.07468070089817047,
-      "rewards/rejected": -0.05860542505979538,
+      "logits/chosen": -3.063744306564331,
+      "logits/rejected": -3.066366195678711,
+      "logps/chosen": -277.1488952636719,
+      "logps/rejected": -88.2572250366211,
+      "loss": 0.4931,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.07289155572652817,
+      "rewards/margins": 0.5126849412918091,
+      "rewards/rejected": -0.4397934079170227,
       "step": 350
     },
     {
       "epoch": 0.74,
       "learning_rate": 1.43348623853211e-07,
-      "logits/chosen": -2.999647855758667,
-      "logits/rejected": -2.999812602996826,
-      "logps/chosen": -260.34814453125,
-      "logps/rejected": -228.0465545654297,
-      "loss": 0.6576,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.010297578759491444,
-      "rewards/margins": 0.07942849397659302,
-      "rewards/rejected": -0.06913091242313385,
+      "logits/chosen": -3.0237436294555664,
+      "logits/rejected": -3.0258359909057617,
+      "logps/chosen": -292.0096740722656,
+      "logps/rejected": -81.93167114257812,
+      "loss": 0.4951,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": 0.07367613166570663,
+      "rewards/margins": 0.49797001481056213,
+      "rewards/rejected": -0.4242939352989197,
       "step": 360
     },
     {
       "epoch": 0.76,
       "learning_rate": 1.318807339449541e-07,
-      "logits/chosen": -3.029534101486206,
-      "logits/rejected": -3.0314173698425293,
-      "logps/chosen": -284.08721923828125,
-      "logps/rejected": -248.7538604736328,
-      "loss": 0.6532,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.036643363535404205,
-      "rewards/margins": 0.10196901857852936,
-      "rewards/rejected": -0.06532564014196396,
+      "logits/chosen": -2.9882092475891113,
+      "logits/rejected": -2.9637956619262695,
+      "logps/chosen": -274.551513671875,
+      "logps/rejected": -73.8973388671875,
+      "loss": 0.496,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.0880483016371727,
+      "rewards/margins": 0.49274787306785583,
+      "rewards/rejected": -0.4046996533870697,
       "step": 370
     },
     {
       "epoch": 0.78,
       "learning_rate": 1.2041284403669725e-07,
-      "logits/chosen": -3.0199809074401855,
-      "logits/rejected": -3.012413501739502,
-      "logps/chosen": -237.28573608398438,
-      "logps/rejected": -243.95590209960938,
-      "loss": 0.6538,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.012126882560551167,
-      "rewards/margins": 0.08107715100049973,
-      "rewards/rejected": -0.06895027309656143,
+      "logits/chosen": -3.070621967315674,
+      "logits/rejected": -3.0683789253234863,
+      "logps/chosen": -266.607177734375,
+      "logps/rejected": -81.02775573730469,
+      "loss": 0.493,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.10891600698232651,
+      "rewards/margins": 0.5303564071655273,
+      "rewards/rejected": -0.42144036293029785,
       "step": 380
     },
     {
       "epoch": 0.8,
       "learning_rate": 1.0894495412844036e-07,
-      "logits/chosen": -2.9979848861694336,
-      "logits/rejected": -3.015382766723633,
-      "logps/chosen": -312.633544921875,
-      "logps/rejected": -235.29806518554688,
-      "loss": 0.6503,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.013257297687232494,
-      "rewards/margins": 0.09175875037908554,
-      "rewards/rejected": -0.07850147038698196,
+      "logits/chosen": -3.0497114658355713,
+      "logits/rejected": -3.053192615509033,
+      "logps/chosen": -280.43218994140625,
+      "logps/rejected": -80.42735290527344,
+      "loss": 0.4892,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10893626511096954,
+      "rewards/margins": 0.5605167746543884,
+      "rewards/rejected": -0.4515805244445801,
       "step": 390
     },
     {
       "epoch": 0.82,
       "learning_rate": 9.747706422018348e-08,
-      "logits/chosen": -3.048086166381836,
-      "logits/rejected": -3.0504872798919678,
-      "logps/chosen": -278.25189208984375,
-      "logps/rejected": -248.26510620117188,
-      "loss": 0.6511,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.01932488940656185,
-      "rewards/margins": 0.07586248964071274,
-      "rewards/rejected": -0.05653759837150574,
+      "logits/chosen": -3.002933979034424,
+      "logits/rejected": -3.0063657760620117,
+      "logps/chosen": -241.24276733398438,
+      "logps/rejected": -75.92924499511719,
+      "loss": 0.4833,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 0.07781459391117096,
+      "rewards/margins": 0.46425342559814453,
+      "rewards/rejected": -0.38643890619277954,
       "step": 400
     },
     {
       "epoch": 0.85,
       "learning_rate": 8.60091743119266e-08,
-      "logits/chosen": -3.0442748069763184,
-      "logits/rejected": -3.0469086170196533,
-      "logps/chosen": -291.3175048828125,
-      "logps/rejected": -228.79153442382812,
-      "loss": 0.6492,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.026980062946677208,
-      "rewards/margins": 0.10874257236719131,
-      "rewards/rejected": -0.08176250755786896,
+      "logits/chosen": -3.0454163551330566,
+      "logits/rejected": -3.035583972930908,
+      "logps/chosen": -264.18585205078125,
+      "logps/rejected": -78.031982421875,
+      "loss": 0.4744,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.09802711009979248,
+      "rewards/margins": 0.5436574816703796,
+      "rewards/rejected": -0.44563040137290955,
       "step": 410
     },
     {
       "epoch": 0.87,
       "learning_rate": 7.454128440366971e-08,
-      "logits/chosen": -3.0467171669006348,
-      "logits/rejected": -3.0507471561431885,
-      "logps/chosen": -274.8234558105469,
-      "logps/rejected": -227.38638305664062,
-      "loss": 0.6509,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.021679330617189407,
-      "rewards/margins": 0.10154237598180771,
-      "rewards/rejected": -0.0798630565404892,
+      "logits/chosen": -3.0196666717529297,
+      "logits/rejected": -3.0026302337646484,
+      "logps/chosen": -272.02630615234375,
+      "logps/rejected": -82.01240539550781,
+      "loss": 0.481,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 0.08279488980770111,
+      "rewards/margins": 0.5704164505004883,
+      "rewards/rejected": -0.48762160539627075,
       "step": 420
     },
     {
       "epoch": 0.89,
       "learning_rate": 6.307339449541284e-08,
-      "logits/chosen": -3.0327906608581543,
-      "logits/rejected": -3.0403802394866943,
-      "logps/chosen": -256.6832580566406,
-      "logps/rejected": -259.84295654296875,
-      "loss": 0.651,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.014505205675959587,
-      "rewards/margins": 0.08562619239091873,
-      "rewards/rejected": -0.07112099230289459,
+      "logits/chosen": -3.0509345531463623,
+      "logits/rejected": -3.0137345790863037,
+      "logps/chosen": -262.2018127441406,
+      "logps/rejected": -77.63418579101562,
+      "loss": 0.4731,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.1073322519659996,
+      "rewards/margins": 0.5776056051254272,
+      "rewards/rejected": -0.4702734053134918,
       "step": 430
     },
     {
       "epoch": 0.91,
       "learning_rate": 5.1605504587155966e-08,
-      "logits/chosen": -3.0034422874450684,
-      "logits/rejected": -3.013791799545288,
-      "logps/chosen": -301.6542053222656,
-      "logps/rejected": -234.38381958007812,
-      "loss": 0.6493,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.01893182098865509,
-      "rewards/margins": 0.12277624756097794,
-      "rewards/rejected": -0.10384440422058105,
+      "logits/chosen": -3.0285000801086426,
+      "logits/rejected": -3.0236475467681885,
+      "logps/chosen": -266.83599853515625,
+      "logps/rejected": -77.38362121582031,
+      "loss": 0.476,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 0.08291526138782501,
+      "rewards/margins": 0.4984784722328186,
+      "rewards/rejected": -0.41556310653686523,
       "step": 440
     },
     {
       "epoch": 0.93,
       "learning_rate": 4.0137614678899086e-08,
-      "logits/chosen": -3.0094974040985107,
-      "logits/rejected": -2.9743194580078125,
-      "logps/chosen": -290.7796630859375,
-      "logps/rejected": -238.8968963623047,
-      "loss": 0.6521,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.029885241761803627,
-      "rewards/margins": 0.11550422757863998,
-      "rewards/rejected": -0.08561898022890091,
+      "logits/chosen": -3.02640438079834,
+      "logits/rejected": -3.011373996734619,
+      "logps/chosen": -295.5868835449219,
+      "logps/rejected": -80.76414489746094,
+      "loss": 0.4707,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.09663239866495132,
+      "rewards/margins": 0.5815601944923401,
+      "rewards/rejected": -0.48492780327796936,
       "step": 450
     },
     {
       "epoch": 0.95,
       "learning_rate": 2.86697247706422e-08,
-      "logits/chosen": -3.043740749359131,
-      "logits/rejected": -3.035067081451416,
-      "logps/chosen": -258.144287109375,
-      "logps/rejected": -234.55081176757812,
-      "loss": 0.6483,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.01103687472641468,
-      "rewards/margins": 0.0873931497335434,
-      "rewards/rejected": -0.07635627686977386,
+      "logits/chosen": -3.0195059776306152,
+      "logits/rejected": -2.988323926925659,
+      "logps/chosen": -300.5026550292969,
+      "logps/rejected": -86.79838562011719,
+      "loss": 0.4808,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.11054690927267075,
+      "rewards/margins": 0.5899176001548767,
+      "rewards/rejected": -0.47937074303627014,
       "step": 460
     },
     {
       "epoch": 0.97,
       "learning_rate": 1.720183486238532e-08,
-      "logits/chosen": -3.0199711322784424,
-      "logits/rejected": -3.026895761489868,
-      "logps/chosen": -288.91119384765625,
-      "logps/rejected": -242.3592071533203,
-      "loss": 0.6511,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.03277132660150528,
-      "rewards/margins": 0.10831280797719955,
-      "rewards/rejected": -0.07554147392511368,
+      "logits/chosen": -3.0426931381225586,
+      "logits/rejected": -3.0394179821014404,
+      "logps/chosen": -235.52706909179688,
+      "logps/rejected": -73.9857406616211,
+      "loss": 0.4819,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 0.08785500377416611,
+      "rewards/margins": 0.5274263620376587,
+      "rewards/rejected": -0.4395713806152344,
       "step": 470
     },
     {
       "epoch": 0.99,
       "learning_rate": 5.73394495412844e-09,
-      "logits/chosen": -3.041350841522217,
-      "logits/rejected": -3.058216094970703,
-      "logps/chosen": -258.9853820800781,
-      "logps/rejected": -226.7718048095703,
-      "loss": 0.6488,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.022982869297266006,
-      "rewards/margins": 0.09263849258422852,
-      "rewards/rejected": -0.06965561956167221,
+      "logits/chosen": -3.0092616081237793,
+      "logits/rejected": -2.972731590270996,
+      "logps/chosen": -249.88876342773438,
+      "logps/rejected": -85.80451965332031,
+      "loss": 0.482,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.07512323558330536,
+      "rewards/margins": 0.5230099558830261,
+      "rewards/rejected": -0.44788676500320435,
       "step": 480
     },
     {
       "epoch": 1.0,
-      "eval_logits/chosen": -3.07612943649292,
-      "eval_logits/rejected": -3.056239604949951,
-      "eval_logps/chosen": -271.6427917480469,
-      "eval_logps/rejected": -224.8079376220703,
-      "eval_loss": 0.6488261818885803,
-      "eval_rewards/accuracies": 0.7109375,
-      "eval_rewards/chosen": 0.034067459404468536,
-      "eval_rewards/margins": 0.11610361933708191,
-      "eval_rewards/rejected": -0.08203616738319397,
-      "eval_runtime": 255.1726,
-      "eval_samples_per_second": 7.838,
-      "eval_steps_per_second": 0.063,
+      "eval_logits/chosen": -3.034407377243042,
+      "eval_logits/rejected": -3.069913864135742,
+      "eval_logps/chosen": -271.40020751953125,
+      "eval_logps/rejected": -175.5244140625,
+      "eval_loss": 0.5650191903114319,
+      "eval_rewards/accuracies": 0.76953125,
+      "eval_rewards/chosen": 0.08157022297382355,
+      "eval_rewards/margins": 0.33799096941947937,
+      "eval_rewards/rejected": -0.25642073154449463,
+      "eval_runtime": 256.4523,
+      "eval_samples_per_second": 7.799,
+      "eval_steps_per_second": 0.062,
       "step": 485
     },
     {
       "epoch": 1.0,
       "step": 485,
       "total_flos": 0.0,
-      "train_loss": 0.6667533972828659,
-      "train_runtime": 15505.6746,
-      "train_samples_per_second": 3.996,
+      "train_loss": 0.5539181610972611,
+      "train_runtime": 15602.6148,
+      "train_samples_per_second": 3.978,
       "train_steps_per_second": 0.031
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0e4c37c4ba74a7c42b124e93bdaa61e543cd3533851bf4e87301d3ef2e466cd
+oid sha256:178ca2e9659218d5d6a040bc9b839def7f1e2d04d37ea8a694c883bf79442126
 size 5688