Model save

README.md

@@ -2,13 +2,9 @@
 license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - dpo
 - generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 base_model: mistralai/Mistral-7B-v0.1
 model-index:
 - name: zephyr-7b-dpo-qlora
@@ -20,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-qlora
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-qlora](https://huggingface.co/alignment-handbook/zephyr-7b-sft-qlora) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2121.0452
-- Rewards/chosen: 0.0578
-- Rewards/rejected: -0.0912
-- Rewards/accuracies: 0.7599
-- Rewards/margins: 0.1490
-- Logps/rejected: -253.8891
-- Logps/chosen: -259.2458
-- Logits/rejected: -2.2028
-- Logits/chosen: -2.2552
+- Loss: 1721.1201
+- Rewards/chosen: -0.0627
+- Rewards/rejected: -0.2250
+- Rewards/accuracies: 0.7738
+- Rewards/margins: 0.1623
+- Logps/rejected: -267.2721
+- Logps/chosen: -271.2979
+- Logits/rejected: -2.0354
+- Logits/chosen: -2.0918
 
 ## Model description
 
@@ -67,10 +63,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 2149.4746     | 0.21  | 100  | 2190.7666       | 0.0445         | -0.0848          | 0.7460             | 0.1293          | -253.2523      | -260.5782    | -2.1770         | -2.2229       |
-| 2105.1256     | 0.42  | 200  | 2151.1555       | 0.0543         | -0.0961          | 0.7599             | 0.1504          | -254.3840      | -259.5941    | -2.2074         | -2.2603       |
-| 2135.4973     | 0.63  | 300  | 2129.0896       | 0.0626         | -0.0799          | 0.7560             | 0.1425          | -252.7585      | -258.7624    | -2.2232         | -2.2765       |
-| 2099.8018     | 0.84  | 400  | 2121.6672       | 0.0538         | -0.0959          | 0.7540             | 0.1497          | -254.3591      | -259.6440    | -2.2016         | -2.2541       |
+| 1797.9404     | 0.21  | 100  | 1887.4103       | 0.0131         | -0.1197          | 0.7520             | 0.1328          | -256.7424      | -263.7133    | -2.1486         | -2.1969       |
+| 1700.9055     | 0.42  | 200  | 1784.6598       | -0.0464        | -0.2062          | 0.7619             | 0.1598          | -265.3905      | -269.6655    | -2.1081         | -2.1618       |
+| 1767.2219     | 0.63  | 300  | 1735.5183       | -0.0467        | -0.2001          | 0.7698             | 0.1534          | -264.7795      | -269.6956    | -2.1057         | -2.1587       |
+| 1717.4336     | 0.84  | 400  | 1721.6765       | -0.0691        | -0.2309          | 0.7718             | 0.1618          | -267.8569      | -271.9333    | -2.0322         | -2.0885       |
 
 
 ### Framework versions

adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "up_proj",
-    "v_proj",
     "q_proj",
     "gate_proj",
+    "up_proj",
+    "k_proj",
     "down_proj",
     "o_proj",
-    "k_proj"
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6f7d1a95415a6c03799926b1b3b0647c3602207bcc4fb5c48fa957c5b2fea04
+oid sha256:58513bf1529e315eda3b88d4c9cacb2897ba3fd8a6c935b6b16975253aa6b856
 size 671150064

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.255185127258301,
-    "eval_logits/rejected": -2.2027812004089355,
-    "eval_logps/chosen": -259.245849609375,
-    "eval_logps/rejected": -253.8891143798828,
-    "eval_loss": 2121.045166015625,
-    "eval_rewards/accuracies": 0.7599206566810608,
-    "eval_rewards/chosen": 0.057787273079156876,
-    "eval_rewards/margins": 0.14896924793720245,
-    "eval_rewards/rejected": -0.09118196368217468,
-    "eval_runtime": 547.839,
+    "eval_logits/chosen": -2.0918362140655518,
+    "eval_logits/rejected": -2.03544020652771,
+    "eval_logps/chosen": -271.2979431152344,
+    "eval_logps/rejected": -267.2720642089844,
+    "eval_loss": 1721.1201171875,
+    "eval_rewards/accuracies": 0.773809552192688,
+    "eval_rewards/chosen": -0.06273359060287476,
+    "eval_rewards/margins": 0.16227789223194122,
+    "eval_rewards/rejected": -0.22501146793365479,
+    "eval_runtime": 548.8776,
     "eval_samples": 2000,
-    "eval_samples_per_second": 3.651,
+    "eval_samples_per_second": 3.644,
     "eval_steps_per_second": 0.115,
-    "train_loss": 2164.5614415454666,
-    "train_runtime": 32346.8016,
+    "train_loss": 1826.8015694608227,
+    "train_runtime": 32379.7062,
     "train_samples": 61135,
-    "train_samples_per_second": 1.89,
+    "train_samples_per_second": 1.888,
     "train_steps_per_second": 0.015
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.255185127258301,
-    "eval_logits/rejected": -2.2027812004089355,
-    "eval_logps/chosen": -259.245849609375,
-    "eval_logps/rejected": -253.8891143798828,
-    "eval_loss": 2121.045166015625,
-    "eval_rewards/accuracies": 0.7599206566810608,
-    "eval_rewards/chosen": 0.057787273079156876,
-    "eval_rewards/margins": 0.14896924793720245,
-    "eval_rewards/rejected": -0.09118196368217468,
-    "eval_runtime": 547.839,
+    "eval_logits/chosen": -2.0918362140655518,
+    "eval_logits/rejected": -2.03544020652771,
+    "eval_logps/chosen": -271.2979431152344,
+    "eval_logps/rejected": -267.2720642089844,
+    "eval_loss": 1721.1201171875,
+    "eval_rewards/accuracies": 0.773809552192688,
+    "eval_rewards/chosen": -0.06273359060287476,
+    "eval_rewards/margins": 0.16227789223194122,
+    "eval_rewards/rejected": -0.22501146793365479,
+    "eval_runtime": 548.8776,
     "eval_samples": 2000,
-    "eval_samples_per_second": 3.651,
+    "eval_samples_per_second": 3.644,
     "eval_steps_per_second": 0.115
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 2164.5614415454666,
-    "train_runtime": 32346.8016,
+    "train_loss": 1826.8015694608227,
+    "train_runtime": 32379.7062,
     "train_samples": 61135,
-    "train_samples_per_second": 1.89,
+    "train_samples_per_second": 1.888,
     "train_steps_per_second": 0.015
 }

trainer_state.json

@@ -25,732 +25,732 @@
     {
       "epoch": 0.02,
       "learning_rate": 1.0416666666666667e-06,
-      "logits/chosen": -2.5851330757141113,
-      "logits/rejected": -2.6188478469848633,
-      "logps/chosen": -265.6952209472656,
-      "logps/rejected": -261.4213562011719,
-      "loss": 2495.385,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.005977082531899214,
-      "rewards/margins": 0.0005994850071147084,
-      "rewards/rejected": 0.005377596709877253,
+      "logits/chosen": -2.585383176803589,
+      "logits/rejected": -2.6190898418426514,
+      "logps/chosen": -265.6199035644531,
+      "logps/rejected": -261.3590393066406,
+      "loss": 2489.4685,
+      "rewards/accuracies": 0.4548611044883728,
+      "rewards/chosen": 0.006730278953909874,
+      "rewards/margins": 0.0007296364055946469,
+      "rewards/rejected": 0.006000642664730549,
       "step": 10
     },
     {
       "epoch": 0.04,
       "learning_rate": 2.0833333333333334e-06,
-      "logits/chosen": -2.6101512908935547,
-      "logits/rejected": -2.5939109325408936,
-      "logps/chosen": -255.68185424804688,
-      "logps/rejected": -248.1254119873047,
-      "loss": 2457.86,
-      "rewards/accuracies": 0.628125011920929,
-      "rewards/chosen": 0.013690793886780739,
-      "rewards/margins": 0.00916606467217207,
-      "rewards/rejected": 0.004524729214608669,
+      "logits/chosen": -2.616151809692383,
+      "logits/rejected": -2.599904775619507,
+      "logps/chosen": -253.3858184814453,
+      "logps/rejected": -245.82345581054688,
+      "loss": 2411.3754,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.036651305854320526,
+      "rewards/margins": 0.009106594137847424,
+      "rewards/rejected": 0.02754470705986023,
       "step": 20
     },
     {
       "epoch": 0.06,
       "learning_rate": 3.125e-06,
-      "logits/chosen": -2.604323148727417,
-      "logits/rejected": -2.598053455352783,
-      "logps/chosen": -254.423095703125,
-      "logps/rejected": -226.73153686523438,
-      "loss": 2402.3988,
-      "rewards/accuracies": 0.703125,
-      "rewards/chosen": 0.01266755722463131,
-      "rewards/margins": 0.024019470438361168,
-      "rewards/rejected": -0.01135191135108471,
+      "logits/chosen": -2.617845058441162,
+      "logits/rejected": -2.6118521690368652,
+      "logps/chosen": -250.7469482421875,
+      "logps/rejected": -223.05172729492188,
+      "loss": 2306.1311,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": 0.04942930489778519,
+      "rewards/margins": 0.023983022198081017,
+      "rewards/rejected": 0.02544628083705902,
       "step": 30
     },
     {
       "epoch": 0.08,
       "learning_rate": 4.166666666666667e-06,
-      "logits/chosen": -2.6043972969055176,
-      "logits/rejected": -2.582412004470825,
-      "logps/chosen": -279.12042236328125,
-      "logps/rejected": -241.2065887451172,
-      "loss": 2290.4264,
+      "logits/chosen": -2.6323208808898926,
+      "logits/rejected": -2.608524799346924,
+      "logps/chosen": -276.45947265625,
+      "logps/rejected": -238.35391235351562,
+      "loss": 2100.6182,
       "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": 0.024520257487893105,
-      "rewards/margins": 0.0557018406689167,
-      "rewards/rejected": -0.031181585043668747,
+      "rewards/chosen": 0.05112973973155022,
+      "rewards/margins": 0.05378426983952522,
+      "rewards/rejected": -0.002654529409483075,
       "step": 40
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999731868769027e-06,
-      "logits/chosen": -2.531161308288574,
-      "logits/rejected": -2.5264387130737305,
-      "logps/chosen": -252.51846313476562,
-      "logps/rejected": -247.7227325439453,
-      "loss": 2291.9322,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.029673133045434952,
-      "rewards/margins": 0.08245684206485748,
-      "rewards/rejected": -0.05278371647000313,
+      "logits/chosen": -2.552873373031616,
+      "logits/rejected": -2.5477213859558105,
+      "logps/chosen": -253.2111358642578,
+      "logps/rejected": -248.1074676513672,
+      "loss": 2103.8223,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": 0.022746428847312927,
+      "rewards/margins": 0.07937721163034439,
+      "rewards/rejected": -0.05663077160716057,
       "step": 50
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.9903533134293035e-06,
-      "logits/chosen": -2.545037031173706,
-      "logits/rejected": -2.5416412353515625,
-      "logps/chosen": -260.83905029296875,
-      "logps/rejected": -239.8417205810547,
-      "loss": 2269.9371,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.03231300041079521,
-      "rewards/margins": 0.09112317860126495,
-      "rewards/rejected": -0.05881017446517944,
+      "logits/chosen": -2.556926727294922,
+      "logits/rejected": -2.551504611968994,
+      "logps/chosen": -261.6982116699219,
+      "logps/rejected": -240.27059936523438,
+      "loss": 2054.3434,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": 0.023721303790807724,
+      "rewards/margins": 0.08682042360305786,
+      "rewards/rejected": -0.06309913098812103,
       "step": 60
     },
     {
       "epoch": 0.15,
       "learning_rate": 4.967625656594782e-06,
-      "logits/chosen": -2.5832419395446777,
-      "logits/rejected": -2.564356565475464,
-      "logps/chosen": -275.95452880859375,
-      "logps/rejected": -264.7611083984375,
-      "loss": 2236.1113,
-      "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": 0.036882974207401276,
-      "rewards/margins": 0.08578891307115555,
-      "rewards/rejected": -0.048905935138463974,
+      "logits/chosen": -2.5740597248077393,
+      "logits/rejected": -2.553145408630371,
+      "logps/chosen": -278.0965270996094,
+      "logps/rejected": -267.19586181640625,
+      "loss": 1971.1375,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.015462947078049183,
+      "rewards/margins": 0.08871600031852722,
+      "rewards/rejected": -0.07325305044651031,
       "step": 70
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.93167072587771e-06,
-      "logits/chosen": -2.552919864654541,
-      "logits/rejected": -2.524970293045044,
-      "logps/chosen": -257.78448486328125,
-      "logps/rejected": -262.3812561035156,
-      "loss": 2220.0893,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.037375591695308685,
-      "rewards/margins": 0.11339374631643295,
-      "rewards/rejected": -0.07601816952228546,
+      "logits/chosen": -2.5298993587493896,
+      "logits/rejected": -2.5009925365448,
+      "logps/chosen": -258.5903015136719,
+      "logps/rejected": -263.52850341796875,
+      "loss": 1933.0076,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.029317494481801987,
+      "rewards/margins": 0.11680855602025986,
+      "rewards/rejected": -0.08749105781316757,
       "step": 80
     },
     {
       "epoch": 0.19,
       "learning_rate": 4.882681251368549e-06,
-      "logits/chosen": -2.56257963180542,
-      "logits/rejected": -2.5289363861083984,
-      "logps/chosen": -239.4860382080078,
-      "logps/rejected": -252.36196899414062,
-      "loss": 2167.3848,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.04182355850934982,
-      "rewards/margins": 0.10886694490909576,
-      "rewards/rejected": -0.06704337894916534,
+      "logits/chosen": -2.5273003578186035,
+      "logits/rejected": -2.493241548538208,
+      "logps/chosen": -247.47506713867188,
+      "logps/rejected": -260.76678466796875,
+      "loss": 1845.5582,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": -0.03806694597005844,
+      "rewards/margins": 0.11302463710308075,
+      "rewards/rejected": -0.15109160542488098,
       "step": 90
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.8209198325401815e-06,
-      "logits/chosen": -2.5551962852478027,
-      "logits/rejected": -2.562063455581665,
-      "logps/chosen": -266.8739013671875,
-      "logps/rejected": -269.649169921875,
-      "loss": 2149.4746,
+      "logits/chosen": -2.5287628173828125,
+      "logits/rejected": -2.5358829498291016,
+      "logps/chosen": -272.0884704589844,
+      "logps/rejected": -275.2580871582031,
+      "loss": 1797.9404,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.04759662598371506,
-      "rewards/margins": 0.1307816356420517,
-      "rewards/rejected": -0.08318501710891724,
+      "rewards/chosen": -0.0045492262579500675,
+      "rewards/margins": 0.13472509384155273,
+      "rewards/rejected": -0.13927432894706726,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -2.222931385040283,
-      "eval_logits/rejected": -2.1770126819610596,
-      "eval_logps/chosen": -260.57818603515625,
-      "eval_logps/rejected": -253.25228881835938,
-      "eval_loss": 2190.7666015625,
-      "eval_rewards/accuracies": 0.7460317611694336,
-      "eval_rewards/chosen": 0.044464047998189926,
-      "eval_rewards/margins": 0.12927772104740143,
-      "eval_rewards/rejected": -0.0848136618733406,
-      "eval_runtime": 549.355,
-      "eval_samples_per_second": 3.641,
+      "eval_logits/chosen": -2.196876287460327,
+      "eval_logits/rejected": -2.1486356258392334,
+      "eval_logps/chosen": -263.71331787109375,
+      "eval_logps/rejected": -256.7424011230469,
+      "eval_loss": 1887.4102783203125,
+      "eval_rewards/accuracies": 0.7519841194152832,
+      "eval_rewards/chosen": 0.013112416490912437,
+      "eval_rewards/margins": 0.13282696902751923,
+      "eval_rewards/rejected": -0.11971456557512283,
+      "eval_runtime": 549.9966,
+      "eval_samples_per_second": 3.636,
       "eval_steps_per_second": 0.115,
       "step": 100
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.746717530629565e-06,
-      "logits/chosen": -2.5229454040527344,
-      "logits/rejected": -2.5105621814727783,
-      "logps/chosen": -261.46649169921875,
-      "logps/rejected": -256.37835693359375,
-      "loss": 2174.1184,
-      "rewards/accuracies": 0.746874988079071,
-      "rewards/chosen": 0.03517655283212662,
-      "rewards/margins": 0.11897265911102295,
-      "rewards/rejected": -0.08379611372947693,
+      "logits/chosen": -2.480510711669922,
+      "logits/rejected": -2.4668211936950684,
+      "logps/chosen": -267.04180908203125,
+      "logps/rejected": -262.5838317871094,
+      "loss": 1870.9051,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.020576762035489082,
+      "rewards/margins": 0.125274196267128,
+      "rewards/rejected": -0.14585095643997192,
       "step": 110
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.660472094042121e-06,
-      "logits/chosen": -2.5114097595214844,
-      "logits/rejected": -2.481840133666992,
-      "logps/chosen": -246.70370483398438,
-      "logps/rejected": -238.27621459960938,
-      "loss": 2181.3053,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.044524095952510834,
-      "rewards/margins": 0.10293309390544891,
-      "rewards/rejected": -0.05840899422764778,
+      "logits/chosen": -2.44077205657959,
+      "logits/rejected": -2.4053845405578613,
+      "logps/chosen": -256.12939453125,
+      "logps/rejected": -248.28060913085938,
+      "loss": 1855.318,
+      "rewards/accuracies": 0.684374988079071,
+      "rewards/chosen": -0.049732744693756104,
+      "rewards/margins": 0.10872016102075577,
+      "rewards/rejected": -0.15845291316509247,
       "step": 120
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.5626458262912745e-06,
-      "logits/chosen": -2.4726600646972656,
-      "logits/rejected": -2.46514630317688,
-      "logps/chosen": -271.7862548828125,
-      "logps/rejected": -260.61676025390625,
-      "loss": 2175.3252,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.06200919300317764,
-      "rewards/margins": 0.12613125145435333,
-      "rewards/rejected": -0.06412206590175629,
+      "logits/chosen": -2.395805597305298,
+      "logits/rejected": -2.383305311203003,
+      "logps/chosen": -280.74053955078125,
+      "logps/rejected": -270.37860107421875,
+      "loss": 1811.4148,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.027533594518899918,
+      "rewards/margins": 0.1342071145772934,
+      "rewards/rejected": -0.1617407202720642,
       "step": 130
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.453763107901676e-06,
-      "logits/chosen": -2.506436586380005,
-      "logits/rejected": -2.5005128383636475,
-      "logps/chosen": -237.8655242919922,
-      "logps/rejected": -249.9298553466797,
-      "loss": 2167.2516,
+      "logits/chosen": -2.4485344886779785,
+      "logits/rejected": -2.43884015083313,
+      "logps/chosen": -243.1454315185547,
+      "logps/rejected": -255.15432739257812,
+      "loss": 1803.225,
       "rewards/accuracies": 0.734375,
-      "rewards/chosen": 0.024008702486753464,
-      "rewards/margins": 0.1495535969734192,
-      "rewards/rejected": -0.12554487586021423,
+      "rewards/chosen": -0.028790492564439774,
+      "rewards/margins": 0.1489991694688797,
+      "rewards/rejected": -0.17778967320919037,
       "step": 140
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.33440758555951e-06,
-      "logits/chosen": -2.5227842330932617,
-      "logits/rejected": -2.536785364151001,
-      "logps/chosen": -260.7518005371094,
-      "logps/rejected": -235.9630889892578,
-      "loss": 2119.4062,
-      "rewards/accuracies": 0.715624988079071,
-      "rewards/chosen": 0.04733316972851753,
-      "rewards/margins": 0.12345732748508453,
-      "rewards/rejected": -0.0761241465806961,
+      "logits/chosen": -2.459658622741699,
+      "logits/rejected": -2.483065605163574,
+      "logps/chosen": -267.7740478515625,
+      "logps/rejected": -243.34609985351562,
+      "loss": 1781.1752,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.02288922667503357,
+      "rewards/margins": 0.12706486880779266,
+      "rewards/rejected": -0.14995409548282623,
       "step": 150
     },
     {
       "epoch": 0.33,
       "learning_rate": 4.205219043576955e-06,
-      "logits/chosen": -2.5534234046936035,
-      "logits/rejected": -2.4914207458496094,
-      "logps/chosen": -254.14065551757812,
-      "logps/rejected": -250.95700073242188,
-      "loss": 2114.7645,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": 0.06031092256307602,
-      "rewards/margins": 0.15202957391738892,
-      "rewards/rejected": -0.09171866625547409,
+      "logits/chosen": -2.483583688735962,
+      "logits/rejected": -2.4244942665100098,
+      "logps/chosen": -260.3743896484375,
+      "logps/rejected": -258.7478332519531,
+      "loss": 1754.5766,
+      "rewards/accuracies": 0.784375011920929,
+      "rewards/chosen": -0.0020265295170247555,
+      "rewards/margins": 0.16760031878948212,
+      "rewards/rejected": -0.169626846909523,
       "step": 160
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.066889974440757e-06,
-      "logits/chosen": -2.5092320442199707,
-      "logits/rejected": -2.4965577125549316,
-      "logps/chosen": -254.91439819335938,
-      "logps/rejected": -242.8040008544922,
-      "loss": 2229.8135,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.034448813647031784,
-      "rewards/margins": 0.12951095402240753,
-      "rewards/rejected": -0.09506212174892426,
+      "logits/chosen": -2.4374189376831055,
+      "logits/rejected": -2.428433656692505,
+      "logps/chosen": -264.5699768066406,
+      "logps/rejected": -252.79421997070312,
+      "loss": 1953.8818,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.06210694834589958,
+      "rewards/margins": 0.13285748660564423,
+      "rewards/rejected": -0.1949644386768341,
       "step": 170
     },
     {
       "epoch": 0.38,
       "learning_rate": 3.92016186682789e-06,
-      "logits/chosen": -2.521221399307251,
-      "logits/rejected": -2.533686399459839,
-      "logps/chosen": -251.4235382080078,
-      "logps/rejected": -259.76220703125,
-      "loss": 2175.5213,
+      "logits/chosen": -2.467085361480713,
+      "logits/rejected": -2.487204074859619,
+      "logps/chosen": -262.995361328125,
+      "logps/rejected": -271.94183349609375,
+      "loss": 1848.9945,
       "rewards/accuracies": 0.721875011920929,
-      "rewards/chosen": 0.04062749817967415,
-      "rewards/margins": 0.120635487139225,
-      "rewards/rejected": -0.08000798523426056,
+      "rewards/chosen": -0.07509048283100128,
+      "rewards/margins": 0.12671387195587158,
+      "rewards/rejected": -0.20180435478687286,
       "step": 180
     },
     {
       "epoch": 0.4,
       "learning_rate": 3.7658212309857576e-06,
-      "logits/chosen": -2.5192363262176514,
-      "logits/rejected": -2.4917151927948,
-      "logps/chosen": -255.2060089111328,
-      "logps/rejected": -250.82022094726562,
-      "loss": 2099.443,
-      "rewards/accuracies": 0.715624988079071,
-      "rewards/chosen": 0.0492943711578846,
-      "rewards/margins": 0.14053165912628174,
-      "rewards/rejected": -0.09123729914426804,
+      "logits/chosen": -2.450601816177368,
+      "logits/rejected": -2.4304168224334717,
+      "logps/chosen": -269.1886901855469,
+      "logps/rejected": -265.7490539550781,
+      "loss": 1698.6666,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": -0.09053254127502441,
+      "rewards/margins": 0.14999321103096008,
+      "rewards/rejected": -0.2405257225036621,
       "step": 190
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.604695382782159e-06,
-      "logits/chosen": -2.5251801013946533,
-      "logits/rejected": -2.5034642219543457,
-      "logps/chosen": -269.3675537109375,
-      "logps/rejected": -262.86376953125,
-      "loss": 2105.1256,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.0575677752494812,
-      "rewards/margins": 0.14340198040008545,
-      "rewards/rejected": -0.08583419024944305,
+      "logits/chosen": -2.447007179260254,
+      "logits/rejected": -2.419039726257324,
+      "logps/chosen": -282.8253479003906,
+      "logps/rejected": -278.14508056640625,
+      "loss": 1700.9055,
+      "rewards/accuracies": 0.784375011920929,
+      "rewards/chosen": -0.07701022177934647,
+      "rewards/margins": 0.16163742542266846,
+      "rewards/rejected": -0.23864765465259552,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -2.260270833969116,
-      "eval_logits/rejected": -2.2073864936828613,
-      "eval_logps/chosen": -259.5941467285156,
-      "eval_logps/rejected": -254.3839874267578,
-      "eval_loss": 2151.155517578125,
-      "eval_rewards/accuracies": 0.7599206566810608,
-      "eval_rewards/chosen": 0.05430443957448006,
-      "eval_rewards/margins": 0.15043501555919647,
-      "eval_rewards/rejected": -0.09613056480884552,
-      "eval_runtime": 548.195,
-      "eval_samples_per_second": 3.648,
+      "eval_logits/chosen": -2.161839485168457,
+      "eval_logits/rejected": -2.1081268787384033,
+      "eval_logps/chosen": -269.66546630859375,
+      "eval_logps/rejected": -265.3905029296875,
+      "eval_loss": 1784.6597900390625,
+      "eval_rewards/accuracies": 0.761904776096344,
+      "eval_rewards/chosen": -0.0464087538421154,
+      "eval_rewards/margins": 0.15978708863258362,
+      "eval_rewards/rejected": -0.2061958611011505,
+      "eval_runtime": 549.0189,
+      "eval_samples_per_second": 3.643,
       "eval_steps_per_second": 0.115,
       "step": 200
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.437648009023905e-06,
-      "logits/chosen": -2.533383369445801,
-      "logits/rejected": -2.4935860633850098,
-      "logps/chosen": -243.6236114501953,
-      "logps/rejected": -238.85140991210938,
-      "loss": 2145.5416,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.06410142779350281,
-      "rewards/margins": 0.14374245703220367,
-      "rewards/rejected": -0.07964102178812027,
+      "logits/chosen": -2.458688259124756,
+      "logits/rejected": -2.4217796325683594,
+      "logps/chosen": -252.5647430419922,
+      "logps/rejected": -248.326416015625,
+      "loss": 1806.0594,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.02530970238149166,
+      "rewards/margins": 0.14908090233802795,
+      "rewards/rejected": -0.17439061403274536,
       "step": 210
     },
     {
       "epoch": 0.46,
       "learning_rate": 3.265574537815398e-06,
-      "logits/chosen": -2.554565906524658,
-      "logits/rejected": -2.56289005279541,
-      "logps/chosen": -277.4061584472656,
-      "logps/rejected": -253.40048217773438,
-      "loss": 2196.8484,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.052330613136291504,
-      "rewards/margins": 0.11339585483074188,
-      "rewards/rejected": -0.06106524541974068,
+      "logits/chosen": -2.4742610454559326,
+      "logits/rejected": -2.4789376258850098,
+      "logps/chosen": -286.0444030761719,
+      "logps/rejected": -261.9767150878906,
+      "loss": 1855.6273,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.03405206650495529,
+      "rewards/margins": 0.11277566105127335,
+      "rewards/rejected": -0.14682772755622864,
       "step": 220
     },
     {
       "epoch": 0.48,
       "learning_rate": 3.089397338773569e-06,
-      "logits/chosen": -2.4857611656188965,
-      "logits/rejected": -2.473193407058716,
-      "logps/chosen": -247.3427276611328,
-      "logps/rejected": -241.8627471923828,
-      "loss": 2160.1729,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.03845102712512016,
-      "rewards/margins": 0.11976752430200577,
-      "rewards/rejected": -0.0813164934515953,
+      "logits/chosen": -2.38773775100708,
+      "logits/rejected": -2.3718185424804688,
+      "logps/chosen": -257.7181701660156,
+      "logps/rejected": -253.3428955078125,
+      "loss": 1797.9486,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": -0.06530335545539856,
+      "rewards/margins": 0.1308148354291916,
+      "rewards/rejected": -0.19611820578575134,
       "step": 230
     },
     {
       "epoch": 0.5,
       "learning_rate": 2.9100607788275547e-06,
-      "logits/chosen": -2.5121560096740723,
-      "logits/rejected": -2.516338586807251,
-      "logps/chosen": -257.1769714355469,
-      "logps/rejected": -247.3695068359375,
-      "loss": 2185.7641,
-      "rewards/accuracies": 0.684374988079071,
-      "rewards/chosen": 0.0379050187766552,
-      "rewards/margins": 0.11140499264001846,
-      "rewards/rejected": -0.07349997013807297,
+      "logits/chosen": -2.4125852584838867,
+      "logits/rejected": -2.414628267288208,
+      "logps/chosen": -265.2156066894531,
+      "logps/rejected": -257.0289001464844,
+      "loss": 1850.0729,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": -0.04248107224702835,
+      "rewards/margins": 0.12761279940605164,
+      "rewards/rejected": -0.17009387910366058,
       "step": 240
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.72852616010567e-06,
-      "logits/chosen": -2.5092978477478027,
-      "logits/rejected": -2.487090826034546,
-      "logps/chosen": -264.5955505371094,
-      "logps/rejected": -246.3382110595703,
-      "loss": 2136.6197,
-      "rewards/accuracies": 0.7406250238418579,
-      "rewards/chosen": 0.039962492883205414,
-      "rewards/margins": 0.1403963267803192,
-      "rewards/rejected": -0.1004338413476944,
+      "logits/chosen": -2.4339253902435303,
+      "logits/rejected": -2.4054951667785645,
+      "logps/chosen": -271.8371276855469,
+      "logps/rejected": -255.33438110351562,
+      "loss": 1766.1885,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": -0.0324532687664032,
+      "rewards/margins": 0.1579422652721405,
+      "rewards/rejected": -0.1903955340385437,
       "step": 250
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.5457665670441937e-06,
-      "logits/chosen": -2.5069711208343506,
-      "logits/rejected": -2.5030505657196045,
-      "logps/chosen": -257.4859619140625,
-      "logps/rejected": -231.91958618164062,
-      "loss": 2085.2795,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.05723271518945694,
-      "rewards/margins": 0.15024301409721375,
-      "rewards/rejected": -0.0930103212594986,
+      "logits/chosen": -2.4216437339782715,
+      "logits/rejected": -2.4156367778778076,
+      "logps/chosen": -266.7996520996094,
+      "logps/rejected": -243.180419921875,
+      "loss": 1710.8809,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": -0.035904210060834885,
+      "rewards/margins": 0.16971439123153687,
+      "rewards/rejected": -0.20561861991882324,
       "step": 260
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3627616503391813e-06,
-      "logits/chosen": -2.525665760040283,
-      "logits/rejected": -2.5043163299560547,
-      "logps/chosen": -280.7471618652344,
-      "logps/rejected": -267.36712646484375,
-      "loss": 2089.859,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.05569761246442795,
-      "rewards/margins": 0.179846853017807,
-      "rewards/rejected": -0.12414924055337906,
+      "logits/chosen": -2.4438915252685547,
+      "logits/rejected": -2.416748285293579,
+      "logps/chosen": -290.58453369140625,
+      "logps/rejected": -277.0739440917969,
+      "loss": 1714.5062,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.042676471173763275,
+      "rewards/margins": 0.17854078114032745,
+      "rewards/rejected": -0.22121724486351013,
       "step": 270
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.1804923757009885e-06,
-      "logits/chosen": -2.500837564468384,
-      "logits/rejected": -2.501950740814209,
-      "logps/chosen": -270.04193115234375,
-      "logps/rejected": -248.61978149414062,
-      "loss": 2111.6906,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.05320361256599426,
-      "rewards/margins": 0.1410333216190338,
-      "rewards/rejected": -0.08782971650362015,
+      "logits/chosen": -2.414602756500244,
+      "logits/rejected": -2.4200820922851562,
+      "logps/chosen": -282.95147705078125,
+      "logps/rejected": -261.1886291503906,
+      "loss": 1764.4607,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.07589195668697357,
+      "rewards/margins": 0.13762618601322174,
+      "rewards/rejected": -0.2135181427001953,
       "step": 280
     },
     {
       "epoch": 0.61,
       "learning_rate": 1.9999357655598894e-06,
-      "logits/chosen": -2.5122292041778564,
-      "logits/rejected": -2.50368070602417,
-      "logps/chosen": -258.72686767578125,
-      "logps/rejected": -256.91387939453125,
-      "loss": 2137.0592,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.053160279989242554,
-      "rewards/margins": 0.15454119443893433,
-      "rewards/rejected": -0.10138092190027237,
+      "logits/chosen": -2.430169105529785,
+      "logits/rejected": -2.4057881832122803,
+      "logps/chosen": -265.06805419921875,
+      "logps/rejected": -263.2739562988281,
+      "loss": 1786.2846,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": -0.010251840576529503,
+      "rewards/margins": 0.15472975373268127,
+      "rewards/rejected": -0.16498157382011414,
       "step": 290
     },
     {
       "epoch": 0.63,
       "learning_rate": 1.8220596619089576e-06,
-      "logits/chosen": -2.471623659133911,
-      "logits/rejected": -2.4690403938293457,
-      "logps/chosen": -246.51766967773438,
-      "logps/rejected": -251.79257202148438,
-      "loss": 2135.4973,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.0453377440571785,
-      "rewards/margins": 0.12641170620918274,
-      "rewards/rejected": -0.08107397705316544,
+      "logits/chosen": -2.392138957977295,
+      "logits/rejected": -2.3823294639587402,
+      "logps/chosen": -255.75393676757812,
+      "logps/rejected": -261.84271240234375,
+      "loss": 1767.2219,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.04702477902173996,
+      "rewards/margins": 0.13455010950565338,
+      "rewards/rejected": -0.18157489597797394,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -2.2764506340026855,
-      "eval_logits/rejected": -2.2231767177581787,
-      "eval_logps/chosen": -258.7624206542969,
-      "eval_logps/rejected": -252.75852966308594,
-      "eval_loss": 2129.089599609375,
-      "eval_rewards/accuracies": 0.7559523582458496,
-      "eval_rewards/chosen": 0.06262180209159851,
-      "eval_rewards/margins": 0.14249789714813232,
-      "eval_rewards/rejected": -0.07987607270479202,
-      "eval_runtime": 547.9938,
-      "eval_samples_per_second": 3.65,
+      "eval_logits/chosen": -2.158698797225952,
+      "eval_logits/rejected": -2.1057095527648926,
+      "eval_logps/chosen": -269.6955871582031,
+      "eval_logps/rejected": -264.77947998046875,
+      "eval_loss": 1735.518310546875,
+      "eval_rewards/accuracies": 0.7698412537574768,
+      "eval_rewards/chosen": -0.04671022295951843,
+      "eval_rewards/margins": 0.15337513387203217,
+      "eval_rewards/rejected": -0.2000853717327118,
+      "eval_runtime": 548.7136,
+      "eval_samples_per_second": 3.645,
       "eval_steps_per_second": 0.115,
       "step": 300
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.647817538357072e-06,
-      "logits/chosen": -2.5041086673736572,
-      "logits/rejected": -2.495436191558838,
-      "logps/chosen": -264.5109558105469,
-      "logps/rejected": -248.3275604248047,
-      "loss": 2107.123,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.05480458214879036,
-      "rewards/margins": 0.13964474201202393,
-      "rewards/rejected": -0.08484016358852386,
+      "logits/chosen": -2.4140188694000244,
+      "logits/rejected": -2.4031002521514893,
+      "logps/chosen": -274.3767395019531,
+      "logps/rejected": -259.40155029296875,
+      "loss": 1673.9693,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": -0.04385297745466232,
+      "rewards/margins": 0.1517268717288971,
+      "rewards/rejected": -0.19557985663414001,
       "step": 310
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.4781433892011132e-06,
-      "logits/chosen": -2.53191876411438,
-      "logits/rejected": -2.4989166259765625,
-      "logps/chosen": -242.36599731445312,
-      "logps/rejected": -243.78067016601562,
-      "loss": 2076.0621,
-      "rewards/accuracies": 0.7718750238418579,
-      "rewards/chosen": 0.05456935614347458,
-      "rewards/margins": 0.14978976547718048,
-      "rewards/rejected": -0.0952204093337059,
+      "logits/chosen": -2.416640520095825,
+      "logits/rejected": -2.370535135269165,
+      "logps/chosen": -252.216064453125,
+      "logps/rejected": -255.1393280029297,
+      "loss": 1673.8594,
+      "rewards/accuracies": 0.7593749761581421,
+      "rewards/chosen": -0.043931327760219574,
+      "rewards/margins": 0.16487570106983185,
+      "rewards/rejected": -0.20880703628063202,
       "step": 320
     },
     {
       "epoch": 0.69,
       "learning_rate": 1.3139467229135999e-06,
-      "logits/chosen": -2.4768006801605225,
-      "logits/rejected": -2.4569873809814453,
-      "logps/chosen": -263.0523681640625,
-      "logps/rejected": -250.5469207763672,
-      "loss": 2112.1141,
-      "rewards/accuracies": 0.734375,
-      "rewards/chosen": 0.044828929007053375,
-      "rewards/margins": 0.13050048053264618,
-      "rewards/rejected": -0.0856715738773346,
+      "logits/chosen": -2.362358570098877,
+      "logits/rejected": -2.3449196815490723,
+      "logps/chosen": -270.876220703125,
+      "logps/rejected": -259.251953125,
+      "loss": 1731.3877,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": -0.03340950980782509,
+      "rewards/margins": 0.13931182026863098,
+      "rewards/rejected": -0.17272132635116577,
       "step": 330
     },
     {
       "epoch": 0.71,
       "learning_rate": 1.1561076868822756e-06,
-      "logits/chosen": -2.5158028602600098,
-      "logits/rejected": -2.5096983909606934,
-      "logps/chosen": -275.6848449707031,
-      "logps/rejected": -246.7259979248047,
-      "loss": 2151.2445,
-      "rewards/accuracies": 0.746874988079071,
-      "rewards/chosen": 0.052164845168590546,
-      "rewards/margins": 0.15314052999019623,
-      "rewards/rejected": -0.10097566992044449,
+      "logits/chosen": -2.3923397064208984,
+      "logits/rejected": -2.384582281112671,
+      "logps/chosen": -284.8360290527344,
+      "logps/rejected": -257.0713806152344,
+      "loss": 1778.2957,
+      "rewards/accuracies": 0.7718750238418579,
+      "rewards/chosen": -0.039347052574157715,
+      "rewards/margins": 0.1650826632976532,
+      "rewards/rejected": -0.20442970097064972,
       "step": 340
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0054723495346484e-06,
-      "logits/chosen": -2.518799304962158,
-      "logits/rejected": -2.4620516300201416,
-      "logps/chosen": -249.27401733398438,
-      "logps/rejected": -218.7183074951172,
-      "loss": 2093.9803,
-      "rewards/accuracies": 0.7593749761581421,
-      "rewards/chosen": 0.0662151575088501,
-      "rewards/margins": 0.14556364715099335,
-      "rewards/rejected": -0.07934850454330444,
+      "logits/chosen": -2.3869528770446777,
+      "logits/rejected": -2.3370375633239746,
+      "logps/chosen": -259.75189208984375,
+      "logps/rejected": -231.13577270507812,
+      "loss": 1665.3461,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.03856384754180908,
+      "rewards/margins": 0.1649591028690338,
+      "rewards/rejected": -0.2035229504108429,
       "step": 350
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.628481651367876e-07,
-      "logits/chosen": -2.5340943336486816,
-      "logits/rejected": -2.5006654262542725,
-      "logps/chosen": -260.32464599609375,
-      "logps/rejected": -237.3218536376953,
-      "loss": 2094.1246,
-      "rewards/accuracies": 0.765625,
-      "rewards/chosen": 0.05396001785993576,
-      "rewards/margins": 0.15317106246948242,
-      "rewards/rejected": -0.09921105206012726,
+      "logits/chosen": -2.4105262756347656,
+      "logits/rejected": -2.352128744125366,
+      "logps/chosen": -269.03790283203125,
+      "logps/rejected": -247.90872192382812,
+      "loss": 1665.3982,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.03317265957593918,
+      "rewards/margins": 0.1719072014093399,
+      "rewards/rejected": -0.20507986843585968,
       "step": 360
     },
     {
       "epoch": 0.77,
       "learning_rate": 7.289996455765749e-07,
-      "logits/chosen": -2.529265880584717,
-      "logits/rejected": -2.515712261199951,
-      "logps/chosen": -266.943115234375,
-      "logps/rejected": -246.0579376220703,
-      "loss": 2115.357,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.052078358829021454,
-      "rewards/margins": 0.1462351232767105,
-      "rewards/rejected": -0.09415675699710846,
+      "logits/chosen": -2.4054064750671387,
+      "logits/rejected": -2.38871431350708,
+      "logps/chosen": -279.2740173339844,
+      "logps/rejected": -259.63690185546875,
+      "loss": 1704.7645,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.07123039662837982,
+      "rewards/margins": 0.15871620178222656,
+      "rewards/rejected": -0.22994661331176758,
       "step": 370
     },
     {
       "epoch": 0.8,
       "learning_rate": 6.046442623320145e-07,
-      "logits/chosen": -2.4891440868377686,
-      "logits/rejected": -2.499753952026367,
-      "logps/chosen": -253.51632690429688,
-      "logps/rejected": -245.4505615234375,
-      "loss": 2082.182,
-      "rewards/accuracies": 0.7406250238418579,
-      "rewards/chosen": 0.051686953753232956,
-      "rewards/margins": 0.1390691101551056,
-      "rewards/rejected": -0.08738215267658234,
+      "logits/chosen": -2.3605639934539795,
+      "logits/rejected": -2.368460178375244,
+      "logps/chosen": -267.2261657714844,
+      "logps/rejected": -260.45550537109375,
+      "loss": 1647.7326,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.0854114517569542,
+      "rewards/margins": 0.15202030539512634,
+      "rewards/rejected": -0.23743176460266113,
       "step": 380
     },
     {
       "epoch": 0.82,
       "learning_rate": 4.904486005914027e-07,
-      "logits/chosen": -2.532160997390747,
-      "logits/rejected": -2.5001654624938965,
-      "logps/chosen": -280.9754333496094,
-      "logps/rejected": -279.0588684082031,
-      "loss": 2114.3043,
-      "rewards/accuracies": 0.7593749761581421,
-      "rewards/chosen": 0.0547635443508625,
-      "rewards/margins": 0.14076778292655945,
-      "rewards/rejected": -0.08600424975156784,
+      "logits/chosen": -2.4076011180877686,
+      "logits/rejected": -2.3770554065704346,
+      "logps/chosen": -292.8500061035156,
+      "logps/rejected": -292.0636291503906,
+      "loss": 1739.5414,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": -0.06398223340511322,
+      "rewards/margins": 0.15206970274448395,
+      "rewards/rejected": -0.21605193614959717,
       "step": 390
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.8702478614051353e-07,
-      "logits/chosen": -2.4791765213012695,
-      "logits/rejected": -2.4799935817718506,
-      "logps/chosen": -246.14102172851562,
-      "logps/rejected": -251.533447265625,
-      "loss": 2099.8018,
-      "rewards/accuracies": 0.703125,
-      "rewards/chosen": 0.0392024889588356,
-      "rewards/margins": 0.13221651315689087,
-      "rewards/rejected": -0.09301402419805527,
+      "logits/chosen": -2.3384757041931152,
+      "logits/rejected": -2.3366100788116455,
+      "logps/chosen": -259.2252502441406,
+      "logps/rejected": -265.5692138671875,
+      "loss": 1717.4336,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.09164019674062729,
+      "rewards/margins": 0.14173154532909393,
+      "rewards/rejected": -0.23337173461914062,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -2.254145860671997,
-      "eval_logits/rejected": -2.2016360759735107,
-      "eval_logps/chosen": -259.64398193359375,
-      "eval_logps/rejected": -254.3590850830078,
-      "eval_loss": 2121.667236328125,
-      "eval_rewards/accuracies": 0.7539682388305664,
-      "eval_rewards/chosen": 0.05380600318312645,
-      "eval_rewards/margins": 0.14968746900558472,
-      "eval_rewards/rejected": -0.09588146954774857,
-      "eval_runtime": 547.9727,
-      "eval_samples_per_second": 3.65,
+      "eval_logits/chosen": -2.088451385498047,
+      "eval_logits/rejected": -2.032222032546997,
+      "eval_logps/chosen": -271.9333190917969,
+      "eval_logps/rejected": -267.85687255859375,
+      "eval_loss": 1721.676513671875,
+      "eval_rewards/accuracies": 0.77182537317276,
+      "eval_rewards/chosen": -0.06908722221851349,
+      "eval_rewards/margins": 0.1617719829082489,
+      "eval_rewards/rejected": -0.23085922002792358,
+      "eval_runtime": 548.423,
+      "eval_samples_per_second": 3.647,
       "eval_steps_per_second": 0.115,
       "step": 400
     },
     {
       "epoch": 0.86,
       "learning_rate": 2.9492720416985004e-07,
-      "logits/chosen": -2.4832329750061035,
-      "logits/rejected": -2.463463306427002,
-      "logps/chosen": -284.7741394042969,
-      "logps/rejected": -252.4269561767578,
-      "loss": 2145.448,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.05263269692659378,
-      "rewards/margins": 0.15021036565303802,
-      "rewards/rejected": -0.09757767617702484,
+      "logits/chosen": -2.3725836277008057,
+      "logits/rejected": -2.3304688930511475,
+      "logps/chosen": -296.72991943359375,
+      "logps/rejected": -266.0842590332031,
+      "loss": 1755.1898,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": -0.0669253021478653,
+      "rewards/margins": 0.16722533106803894,
+      "rewards/rejected": -0.23415064811706543,
       "step": 410
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.1464952759020857e-07,
-      "logits/chosen": -2.4804348945617676,
-      "logits/rejected": -2.457764148712158,
-      "logps/chosen": -254.78604125976562,
-      "logps/rejected": -278.61346435546875,
-      "loss": 2123.6629,
-      "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": 0.033899884670972824,
-      "rewards/margins": 0.11116783320903778,
-      "rewards/rejected": -0.07726795971393585,
+      "logits/chosen": -2.3592472076416016,
+      "logits/rejected": -2.331540107727051,
+      "logps/chosen": -266.99005126953125,
+      "logps/rejected": -292.03680419921875,
+      "loss": 1730.9672,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.08814045041799545,
+      "rewards/margins": 0.12336041778326035,
+      "rewards/rejected": -0.2115008533000946,
       "step": 420
     },
     {
       "epoch": 0.9,
       "learning_rate": 1.4662207078575685e-07,
-      "logits/chosen": -2.4848549365997314,
-      "logits/rejected": -2.485640048980713,
-      "logps/chosen": -268.3457336425781,
-      "logps/rejected": -268.5885925292969,
-      "loss": 2144.4309,
-      "rewards/accuracies": 0.721875011920929,
-      "rewards/chosen": 0.03841588646173477,
-      "rewards/margins": 0.13024446368217468,
-      "rewards/rejected": -0.09182857722043991,
+      "logits/chosen": -2.366381883621216,
+      "logits/rejected": -2.35951566696167,
+      "logps/chosen": -280.3116149902344,
+      "logps/rejected": -281.93939208984375,
+      "loss": 1760.3617,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": -0.08124328404664993,
+      "rewards/margins": 0.14409320056438446,
+      "rewards/rejected": -0.2253364771604538,
       "step": 430
     },
     {
       "epoch": 0.92,
       "learning_rate": 9.120948298936422e-08,
-      "logits/chosen": -2.457054615020752,
-      "logits/rejected": -2.4329726696014404,
-      "logps/chosen": -231.9584197998047,
-      "logps/rejected": -234.6277313232422,
-      "loss": 2118.3984,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.038600482046604156,
-      "rewards/margins": 0.13669805228710175,
-      "rewards/rejected": -0.09809757024049759,
+      "logits/chosen": -2.3266444206237793,
+      "logits/rejected": -2.2898497581481934,
+      "logps/chosen": -243.9964141845703,
+      "logps/rejected": -248.1795196533203,
+      "loss": 1711.7143,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.08177933841943741,
+      "rewards/margins": 0.15183614194393158,
+      "rewards/rejected": -0.23361548781394958,
       "step": 440
     },
     {
       "epoch": 0.94,
       "learning_rate": 4.870879364444109e-08,
-      "logits/chosen": -2.5156655311584473,
-      "logits/rejected": -2.563300848007202,
-      "logps/chosen": -263.9936218261719,
-      "logps/rejected": -265.6227722167969,
-      "loss": 2123.5402,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.04902677983045578,
-      "rewards/margins": 0.1260160207748413,
-      "rewards/rejected": -0.07698923349380493,
+      "logits/chosen": -2.3814821243286133,
+      "logits/rejected": -2.4406635761260986,
+      "logps/chosen": -275.4070129394531,
+      "logps/rejected": -278.91082763671875,
+      "loss": 1743.9877,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": -0.06510698050260544,
+      "rewards/margins": 0.14476314187049866,
+      "rewards/rejected": -0.2098701000213623,
       "step": 450
     },
     {
       "epoch": 0.96,
       "learning_rate": 1.93478202307823e-08,
-      "logits/chosen": -2.470996379852295,
-      "logits/rejected": -2.4720451831817627,
-      "logps/chosen": -258.21734619140625,
-      "logps/rejected": -262.04925537109375,
-      "loss": 2078.5094,
-      "rewards/accuracies": 0.7281249761581421,
-      "rewards/chosen": 0.04391016811132431,
-      "rewards/margins": 0.14817874133586884,
-      "rewards/rejected": -0.10426857322454453,
+      "logits/chosen": -2.34289288520813,
+      "logits/rejected": -2.346625804901123,
+      "logps/chosen": -270.3787841796875,
+      "logps/rejected": -275.61651611328125,
+      "loss": 1676.5176,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.07770398259162903,
+      "rewards/margins": 0.16223737597465515,
+      "rewards/rejected": -0.23994135856628418,
       "step": 460
     },
     {
       "epoch": 0.98,
       "learning_rate": 3.283947088983663e-09,
-      "logits/chosen": -2.513140916824341,
-      "logits/rejected": -2.535651206970215,
-      "logps/chosen": -249.6727752685547,
-      "logps/rejected": -248.2782745361328,
-      "loss": 2093.2779,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.04741714522242546,
-      "rewards/margins": 0.143958181142807,
-      "rewards/rejected": -0.09654103964567184,
+      "logits/chosen": -2.3888649940490723,
+      "logits/rejected": -2.4120144844055176,
+      "logps/chosen": -261.28997802734375,
+      "logps/rejected": -261.67755126953125,
+      "loss": 1663.4154,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.0687546655535698,
+      "rewards/margins": 0.16177912056446075,
+      "rewards/rejected": -0.23053380846977234,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 2164.5614415454666,
-      "train_runtime": 32346.8016,
-      "train_samples_per_second": 1.89,
+      "train_loss": 1826.8015694608227,
+      "train_runtime": 32379.7062,
+      "train_samples_per_second": 1.888,
       "train_steps_per_second": 0.015
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89ae9269821a7a76bfccee733cea8c3af1d1b7b751ef31fb40915f4d080f4944
+oid sha256:e9de8831bf203a26c117251200a242a486dd5bc4f1aae373c17a996f39be3288
 size 4920