Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [Open-Orca/Mistral-7B-OpenOrca](https://huggingface.co/Open-Orca/Mistral-7B-OpenOrca) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.1299
-- Rewards/chosen: -2.1998
-- Rewards/rejected: -6.8894
-- Rewards/accuracies: 0.9521
-- Rewards/margins: 4.6897
-- Logps/rejected: -133.8546
-- Logps/chosen: -85.1177
-- Logits/rejected: -2.5567
-- Logits/chosen: -2.5727
+- Loss: 0.2452
+- Rewards/chosen: -0.7312
+- Rewards/rejected: -2.7785
+- Rewards/accuracies: 0.9132
+- Rewards/margins: 2.0473
+- Logps/rejected: -92.7458
+- Logps/chosen: -70.4321
+- Logits/rejected: -2.6590
+- Logits/chosen: -2.6728
 
 ## Model description
 
@@ -42,7 +42,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-06
+- learning_rate: 3e-06
 - train_batch_size: 1
 - eval_batch_size: 1
 - seed: 42
@@ -52,14 +52,13 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 2
+- num_epochs: 1
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.1534        | 1.0   | 109  | 0.1544          | -1.6908        | -5.4640          | 0.9395             | 3.7732          | -119.6004      | -80.0284     | -2.5938         | -2.6084       |
-| 0.0995        | 1.99  | 218  | 0.1299          | -2.1998        | -6.8894          | 0.9521             | 4.6897          | -133.8546      | -85.1177     | -2.5567         | -2.5727       |
+| 0.2434        | 1.0   | 109  | 0.2452          | -0.7312        | -2.7785          | 0.9132             | 2.0473          | -92.7458       | -70.4321     | -2.6590         | -2.6728       |
 
 
 ### Framework versions

adapter_config.json

@@ -16,8 +16,8 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
     "v_proj",
+    "q_proj",
     "k_proj",
     "o_proj"
   ],

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ded8821d3aefde9251de2b67e8473a6521245ba464d191e6179682e7916ff7e3
+oid sha256:01b81fb6d0db8862e194bf6369b940209b779c3b8b68f5984b4b4c0e6b117ef8
 size 218138576

all_results.json

@@ -1,21 +1,21 @@
 {
-    "epoch": 1.99,
-    "eval_logits/chosen": -2.5726864337921143,
-    "eval_logits/rejected": -2.556718587875366,
-    "eval_logps/chosen": -85.11767578125,
-    "eval_logps/rejected": -133.85464477539062,
-    "eval_loss": 0.12992651760578156,
-    "eval_rewards/accuracies": 0.9520547986030579,
-    "eval_rewards/chosen": -2.199751138687134,
-    "eval_rewards/margins": 4.689663887023926,
-    "eval_rewards/rejected": -6.889414310455322,
-    "eval_runtime": 288.3554,
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.6727941036224365,
+    "eval_logits/rejected": -2.6590042114257812,
+    "eval_logps/chosen": -70.43208312988281,
+    "eval_logps/rejected": -92.74579620361328,
+    "eval_loss": 0.24523001909255981,
+    "eval_rewards/accuracies": 0.913241982460022,
+    "eval_rewards/chosen": -0.7311916947364807,
+    "eval_rewards/margins": 2.0473380088806152,
+    "eval_rewards/rejected": -2.778529405593872,
+    "eval_runtime": 288.4496,
     "eval_samples": 876,
-    "eval_samples_per_second": 3.038,
-    "eval_steps_per_second": 3.038,
-    "train_loss": 0.20939183043777396,
-    "train_runtime": 5162.9911,
+    "eval_samples_per_second": 3.037,
+    "eval_steps_per_second": 3.037,
+    "train_loss": 0.3616804119643815,
+    "train_runtime": 2568.179,
     "train_samples": 3505,
-    "train_samples_per_second": 1.358,
+    "train_samples_per_second": 1.365,
     "train_steps_per_second": 0.042
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 1.99,
-    "eval_logits/chosen": -2.5726864337921143,
-    "eval_logits/rejected": -2.556718587875366,
-    "eval_logps/chosen": -85.11767578125,
-    "eval_logps/rejected": -133.85464477539062,
-    "eval_loss": 0.12992651760578156,
-    "eval_rewards/accuracies": 0.9520547986030579,
-    "eval_rewards/chosen": -2.199751138687134,
-    "eval_rewards/margins": 4.689663887023926,
-    "eval_rewards/rejected": -6.889414310455322,
-    "eval_runtime": 288.3554,
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.6727941036224365,
+    "eval_logits/rejected": -2.6590042114257812,
+    "eval_logps/chosen": -70.43208312988281,
+    "eval_logps/rejected": -92.74579620361328,
+    "eval_loss": 0.24523001909255981,
+    "eval_rewards/accuracies": 0.913241982460022,
+    "eval_rewards/chosen": -0.7311916947364807,
+    "eval_rewards/margins": 2.0473380088806152,
+    "eval_rewards/rejected": -2.778529405593872,
+    "eval_runtime": 288.4496,
     "eval_samples": 876,
-    "eval_samples_per_second": 3.038,
-    "eval_steps_per_second": 3.038
+    "eval_samples_per_second": 3.037,
+    "eval_steps_per_second": 3.037
 }

runs/Dec19_13-43-08_desktop-linux-nlp1/events.out.tfevents.1702968267.desktop-linux-nlp1.3536715.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60466be35623e70dc3691c1104226dc2a80db095f5212e7c9af57deb060ff573
+size 8728

runs/Dec19_14-07-27_desktop-linux-nlp1/events.out.tfevents.1702969726.desktop-linux-nlp1.3564065.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e7926b8e663de8bfdb43712f5d79ebe4c0bed2d49bf25f3b09342bea3775c81
+size 12295

runs/Dec19_14-07-27_desktop-linux-nlp1/events.out.tfevents.1702972583.desktop-linux-nlp1.3564065.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac0fb1689d947adadb16134d5efc503ad3add478f41c6063f111a1a8520d1356
+size 815

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.99,
-    "train_loss": 0.20939183043777396,
-    "train_runtime": 5162.9911,
+    "epoch": 1.0,
+    "train_loss": 0.3616804119643815,
+    "train_runtime": 2568.179,
     "train_samples": 3505,
-    "train_samples_per_second": 1.358,
+    "train_samples_per_second": 1.365,
     "train_steps_per_second": 0.042
 }

trainer_state.json

@@ -1,16 +1,16 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.9902995720399428,
+  "epoch": 0.9951497860199715,
   "eval_steps": 100,
-  "global_step": 218,
+  "global_step": 109,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.01,
-      "learning_rate": 2.2727272727272729e-07,
+      "learning_rate": 2.7272727272727274e-07,
       "logits/chosen": -2.779836893081665,
       "logits/rejected": -2.772892951965332,
       "logps/chosen": -67.39044952392578,
@@ -24,343 +24,173 @@
     },
     {
       "epoch": 0.09,
-      "learning_rate": 2.2727272727272728e-06,
-      "logits/chosen": -2.764805793762207,
-      "logits/rejected": -2.7586262226104736,
-      "logps/chosen": -63.05746841430664,
-      "logps/rejected": -64.96013641357422,
-      "loss": 0.6864,
-      "rewards/accuracies": 0.6006944179534912,
-      "rewards/chosen": 0.009925955906510353,
-      "rewards/margins": 0.013827367685735226,
-      "rewards/rejected": -0.0039014113135635853,
+      "learning_rate": 2.7272727272727272e-06,
+      "logits/chosen": -2.764448881149292,
+      "logits/rejected": -2.7583670616149902,
+      "logps/chosen": -63.044189453125,
+      "logps/rejected": -65.00924682617188,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.6076388955116272,
+      "rewards/chosen": 0.011253755539655685,
+      "rewards/margins": 0.020065149292349815,
+      "rewards/rejected": -0.008811394684016705,
       "step": 10
     },
     {
       "epoch": 0.18,
-      "learning_rate": 4.5454545454545455e-06,
-      "logits/chosen": -2.7655444145202637,
-      "logits/rejected": -2.7531120777130127,
-      "logps/chosen": -60.976318359375,
-      "logps/rejected": -64.35781860351562,
-      "loss": 0.5979,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.06535812467336655,
-      "rewards/margins": 0.22067773342132568,
-      "rewards/rejected": -0.15531960129737854,
+      "learning_rate": 2.7244897959183674e-06,
+      "logits/chosen": -2.7638134956359863,
+      "logits/rejected": -2.7512309551239014,
+      "logps/chosen": -60.88775634765625,
+      "logps/rejected": -64.6989517211914,
+      "loss": 0.5807,
+      "rewards/accuracies": 0.8656250238418579,
+      "rewards/chosen": 0.074214868247509,
+      "rewards/margins": 0.26364782452583313,
+      "rewards/rejected": -0.18943293392658234,
       "step": 20
     },
     {
       "epoch": 0.27,
-      "learning_rate": 4.795918367346939e-06,
-      "logits/chosen": -2.7331104278564453,
-      "logits/rejected": -2.722367763519287,
-      "logps/chosen": -63.54418182373047,
-      "logps/rejected": -77.56448364257812,
-      "loss": 0.3997,
-      "rewards/accuracies": 0.878125011920929,
-      "rewards/chosen": -0.11322204768657684,
-      "rewards/margins": 0.9385285377502441,
-      "rewards/rejected": -1.0517505407333374,
+      "learning_rate": 2.4183673469387754e-06,
+      "logits/chosen": -2.7466235160827637,
+      "logits/rejected": -2.7364418506622314,
+      "logps/chosen": -62.1016845703125,
+      "logps/rejected": -73.9842529296875,
+      "loss": 0.4425,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 0.031027918681502342,
+      "rewards/margins": 0.7247552275657654,
+      "rewards/rejected": -0.6937273740768433,
       "step": 30
     },
     {
       "epoch": 0.37,
-      "learning_rate": 4.540816326530613e-06,
-      "logits/chosen": -2.685359477996826,
-      "logits/rejected": -2.6721653938293457,
-      "logps/chosen": -67.8324966430664,
-      "logps/rejected": -89.94172668457031,
-      "loss": 0.2768,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": -0.5856838226318359,
-      "rewards/margins": 1.7414783239364624,
-      "rewards/rejected": -2.327162265777588,
+      "learning_rate": 2.112244897959184e-06,
+      "logits/chosen": -2.720366954803467,
+      "logits/rejected": -2.707810401916504,
+      "logps/chosen": -64.14788818359375,
+      "logps/rejected": -80.15257263183594,
+      "loss": 0.3545,
+      "rewards/accuracies": 0.909375011920929,
+      "rewards/chosen": -0.217222660779953,
+      "rewards/margins": 1.13102388381958,
+      "rewards/rejected": -1.348246693611145,
       "step": 40
     },
     {
       "epoch": 0.46,
-      "learning_rate": 4.2857142857142855e-06,
-      "logits/chosen": -2.660297393798828,
-      "logits/rejected": -2.6442055702209473,
-      "logps/chosen": -72.59104919433594,
-      "logps/rejected": -93.20745849609375,
-      "loss": 0.2475,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -0.7966881394386292,
-      "rewards/margins": 2.058647632598877,
-      "rewards/rejected": -2.8553357124328613,
+      "learning_rate": 1.806122448979592e-06,
+      "logits/chosen": -2.703882932662964,
+      "logits/rejected": -2.688373327255249,
+      "logps/chosen": -68.16423034667969,
+      "logps/rejected": -81.43087005615234,
+      "loss": 0.3228,
+      "rewards/accuracies": 0.903124988079071,
+      "rewards/chosen": -0.3540056347846985,
+      "rewards/margins": 1.3236706256866455,
+      "rewards/rejected": -1.6776764392852783,
       "step": 50
     },
     {
       "epoch": 0.55,
-      "learning_rate": 4.03061224489796e-06,
-      "logits/chosen": -2.643165111541748,
-      "logits/rejected": -2.63153076171875,
-      "logps/chosen": -72.09125518798828,
-      "logps/rejected": -99.36156463623047,
-      "loss": 0.2392,
-      "rewards/accuracies": 0.903124988079071,
-      "rewards/chosen": -1.083268404006958,
-      "rewards/margins": 2.3197226524353027,
-      "rewards/rejected": -3.4029908180236816,
+      "learning_rate": 1.5e-06,
+      "logits/chosen": -2.6891207695007324,
+      "logits/rejected": -2.6766979694366455,
+      "logps/chosen": -65.70265197753906,
+      "logps/rejected": -84.64549255371094,
+      "loss": 0.3174,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.44440922141075134,
+      "rewards/margins": 1.486973762512207,
+      "rewards/rejected": -1.9313831329345703,
       "step": 60
     },
     {
       "epoch": 0.64,
-      "learning_rate": 3.7755102040816327e-06,
-      "logits/chosen": -2.638352632522583,
-      "logits/rejected": -2.6236445903778076,
-      "logps/chosen": -77.40001678466797,
-      "logps/rejected": -107.3912124633789,
-      "loss": 0.2044,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": -1.5230330228805542,
-      "rewards/margins": 2.718259572982788,
-      "rewards/rejected": -4.241292476654053,
+      "learning_rate": 1.193877551020408e-06,
+      "logits/chosen": -2.696516275405884,
+      "logits/rejected": -2.6814627647399902,
+      "logps/chosen": -68.18341827392578,
+      "logps/rejected": -88.01350402832031,
+      "loss": 0.2817,
+      "rewards/accuracies": 0.903124988079071,
+      "rewards/chosen": -0.6013726592063904,
+      "rewards/margins": 1.7021477222442627,
+      "rewards/rejected": -2.303520441055298,
       "step": 70
     },
     {
       "epoch": 0.73,
-      "learning_rate": 3.5204081632653062e-06,
-      "logits/chosen": -2.6177425384521484,
-      "logits/rejected": -2.597738265991211,
-      "logps/chosen": -75.93782043457031,
-      "logps/rejected": -108.6824951171875,
-      "loss": 0.1696,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -1.198141098022461,
-      "rewards/margins": 3.195936918258667,
-      "rewards/rejected": -4.394078254699707,
+      "learning_rate": 8.877551020408164e-07,
+      "logits/chosen": -2.680356025695801,
+      "logits/rejected": -2.662485122680664,
+      "logps/chosen": -69.7511978149414,
+      "logps/rejected": -89.8312759399414,
+      "loss": 0.2427,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": -0.5794776678085327,
+      "rewards/margins": 1.9294792413711548,
+      "rewards/rejected": -2.5089569091796875,
       "step": 80
     },
     {
       "epoch": 0.82,
-      "learning_rate": 3.2653061224489794e-06,
-      "logits/chosen": -2.6300315856933594,
-      "logits/rejected": -2.6183547973632812,
-      "logps/chosen": -80.39871978759766,
-      "logps/rejected": -113.6379165649414,
-      "loss": 0.1925,
+      "learning_rate": 5.816326530612245e-07,
+      "logits/chosen": -2.6861064434051514,
+      "logits/rejected": -2.6741652488708496,
+      "logps/chosen": -71.44612884521484,
+      "logps/rejected": -92.25032806396484,
+      "loss": 0.257,
       "rewards/accuracies": 0.8968750238418579,
-      "rewards/chosen": -1.5853779315948486,
-      "rewards/margins": 3.1853203773498535,
-      "rewards/rejected": -4.770698547363281,
+      "rewards/chosen": -0.6901184320449829,
+      "rewards/margins": 1.9418220520019531,
+      "rewards/rejected": -2.6319406032562256,
       "step": 90
     },
     {
       "epoch": 0.91,
-      "learning_rate": 3.0102040816326534e-06,
-      "logits/chosen": -2.6064066886901855,
-      "logits/rejected": -2.594722270965576,
-      "logps/chosen": -81.89433288574219,
-      "logps/rejected": -115.00162506103516,
-      "loss": 0.1534,
-      "rewards/accuracies": 0.9468749761581421,
-      "rewards/chosen": -1.6648212671279907,
-      "rewards/margins": 3.4960713386535645,
-      "rewards/rejected": -5.160892486572266,
+      "learning_rate": 2.7551020408163265e-07,
+      "logits/chosen": -2.6638271808624268,
+      "logits/rejected": -2.6519863605499268,
+      "logps/chosen": -72.42386627197266,
+      "logps/rejected": -90.99058532714844,
+      "loss": 0.2434,
+      "rewards/accuracies": 0.9281250238418579,
+      "rewards/chosen": -0.7177737355232239,
+      "rewards/margins": 2.0420150756835938,
+      "rewards/rejected": -2.759788751602173,
       "step": 100
     },
     {
       "epoch": 1.0,
-      "eval_logits/chosen": -2.6083757877349854,
-      "eval_logits/rejected": -2.5937814712524414,
-      "eval_logps/chosen": -80.02838134765625,
-      "eval_logps/rejected": -119.6004409790039,
-      "eval_loss": 0.15436382591724396,
-      "eval_rewards/accuracies": 0.939497709274292,
-      "eval_rewards/chosen": -1.6908209323883057,
-      "eval_rewards/margins": 3.7731716632843018,
-      "eval_rewards/rejected": -5.463992595672607,
-      "eval_runtime": 295.1725,
-      "eval_samples_per_second": 2.968,
-      "eval_steps_per_second": 2.968,
-      "step": 109
-    },
-    {
-      "epoch": 1.0,
-      "learning_rate": 2.7551020408163266e-06,
-      "logits/chosen": -2.6033756732940674,
-      "logits/rejected": -2.582958221435547,
-      "logps/chosen": -80.38322448730469,
-      "logps/rejected": -116.6507797241211,
-      "loss": 0.138,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": -1.656935691833496,
-      "rewards/margins": 3.698695659637451,
-      "rewards/rejected": -5.3556318283081055,
-      "step": 110
-    },
-    {
-      "epoch": 1.1,
-      "learning_rate": 2.5e-06,
-      "logits/chosen": -2.604905843734741,
-      "logits/rejected": -2.591404438018799,
-      "logps/chosen": -80.86669921875,
-      "logps/rejected": -119.6518783569336,
-      "loss": 0.1395,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": -1.803047776222229,
-      "rewards/margins": 3.7810962200164795,
-      "rewards/rejected": -5.58414363861084,
-      "step": 120
-    },
-    {
-      "epoch": 1.19,
-      "learning_rate": 2.244897959183674e-06,
-      "logits/chosen": -2.590282440185547,
-      "logits/rejected": -2.576897144317627,
-      "logps/chosen": -83.53189086914062,
-      "logps/rejected": -131.67037963867188,
-      "loss": 0.1004,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": -1.9269969463348389,
-      "rewards/margins": 4.388735294342041,
-      "rewards/rejected": -6.315732002258301,
-      "step": 130
-    },
-    {
-      "epoch": 1.28,
-      "learning_rate": 1.989795918367347e-06,
-      "logits/chosen": -2.578322410583496,
-      "logits/rejected": -2.5577735900878906,
-      "logps/chosen": -85.14395141601562,
-      "logps/rejected": -126.8256607055664,
-      "loss": 0.1422,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -2.2692408561706543,
-      "rewards/margins": 4.1861348152160645,
-      "rewards/rejected": -6.455375671386719,
-      "step": 140
-    },
-    {
-      "epoch": 1.37,
-      "learning_rate": 1.7346938775510206e-06,
-      "logits/chosen": -2.5812907218933105,
-      "logits/rejected": -2.5663979053497314,
-      "logps/chosen": -87.79288482666016,
-      "logps/rejected": -131.3497314453125,
-      "loss": 0.1284,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": -2.4120209217071533,
-      "rewards/margins": 4.293553352355957,
-      "rewards/rejected": -6.705574035644531,
-      "step": 150
-    },
-    {
-      "epoch": 1.46,
-      "learning_rate": 1.479591836734694e-06,
-      "logits/chosen": -2.596059560775757,
-      "logits/rejected": -2.58156156539917,
-      "logps/chosen": -83.30199432373047,
-      "logps/rejected": -132.67092895507812,
-      "loss": 0.1066,
-      "rewards/accuracies": 0.9593750238418579,
-      "rewards/chosen": -2.065701484680176,
-      "rewards/margins": 4.5842509269714355,
-      "rewards/rejected": -6.6499528884887695,
-      "step": 160
-    },
-    {
-      "epoch": 1.55,
-      "learning_rate": 1.2244897959183673e-06,
-      "logits/chosen": -2.5813376903533936,
-      "logits/rejected": -2.569676160812378,
-      "logps/chosen": -82.94264221191406,
-      "logps/rejected": -132.35789489746094,
-      "loss": 0.0994,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -2.1285476684570312,
-      "rewards/margins": 4.649580955505371,
-      "rewards/rejected": -6.778128147125244,
-      "step": 170
-    },
-    {
-      "epoch": 1.64,
-      "learning_rate": 9.69387755102041e-07,
-      "logits/chosen": -2.575303077697754,
-      "logits/rejected": -2.5673909187316895,
-      "logps/chosen": -86.48652648925781,
-      "logps/rejected": -134.35574340820312,
-      "loss": 0.1224,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": -2.278262138366699,
-      "rewards/margins": 4.526266098022461,
-      "rewards/rejected": -6.80452823638916,
-      "step": 180
-    },
-    {
-      "epoch": 1.73,
-      "learning_rate": 7.142857142857143e-07,
-      "logits/chosen": -2.5673184394836426,
-      "logits/rejected": -2.5392918586730957,
-      "logps/chosen": -85.58756256103516,
-      "logps/rejected": -128.51121520996094,
-      "loss": 0.1345,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": -2.157008409500122,
-      "rewards/margins": 4.34613037109375,
-      "rewards/rejected": -6.503138542175293,
-      "step": 190
-    },
-    {
-      "epoch": 1.83,
-      "learning_rate": 4.591836734693878e-07,
-      "logits/chosen": -2.5774478912353516,
-      "logits/rejected": -2.556039333343506,
-      "logps/chosen": -84.05760955810547,
-      "logps/rejected": -132.11903381347656,
-      "loss": 0.1088,
-      "rewards/accuracies": 0.9593750238418579,
-      "rewards/chosen": -2.128478527069092,
-      "rewards/margins": 4.659018039703369,
-      "rewards/rejected": -6.787497043609619,
-      "step": 200
-    },
-    {
-      "epoch": 1.92,
-      "learning_rate": 2.0408163265306121e-07,
-      "logits/chosen": -2.563249111175537,
-      "logits/rejected": -2.5445797443389893,
-      "logps/chosen": -83.83250427246094,
-      "logps/rejected": -133.15673828125,
-      "loss": 0.0995,
-      "rewards/accuracies": 0.965624988079071,
-      "rewards/chosen": -2.065305709838867,
-      "rewards/margins": 4.883781909942627,
-      "rewards/rejected": -6.949087619781494,
-      "step": 210
-    },
-    {
-      "epoch": 1.99,
-      "eval_logits/chosen": -2.5726864337921143,
-      "eval_logits/rejected": -2.556718587875366,
-      "eval_logps/chosen": -85.11767578125,
-      "eval_logps/rejected": -133.85464477539062,
-      "eval_loss": 0.12992651760578156,
-      "eval_rewards/accuracies": 0.9520547986030579,
-      "eval_rewards/chosen": -2.199751138687134,
-      "eval_rewards/margins": 4.689663887023926,
-      "eval_rewards/rejected": -6.889414310455322,
-      "eval_runtime": 288.5007,
+      "eval_logits/chosen": -2.6727941036224365,
+      "eval_logits/rejected": -2.6590042114257812,
+      "eval_logps/chosen": -70.43208312988281,
+      "eval_logps/rejected": -92.74579620361328,
+      "eval_loss": 0.24523001909255981,
+      "eval_rewards/accuracies": 0.913241982460022,
+      "eval_rewards/chosen": -0.7311916947364807,
+      "eval_rewards/margins": 2.0473380088806152,
+      "eval_rewards/rejected": -2.778529405593872,
+      "eval_runtime": 288.5183,
       "eval_samples_per_second": 3.036,
       "eval_steps_per_second": 3.036,
-      "step": 218
+      "step": 109
     },
     {
-      "epoch": 1.99,
-      "step": 218,
+      "epoch": 1.0,
+      "step": 109,
       "total_flos": 0.0,
-      "train_loss": 0.20939183043777396,
-      "train_runtime": 5162.9911,
-      "train_samples_per_second": 1.358,
+      "train_loss": 0.3616804119643815,
+      "train_runtime": 2568.179,
+      "train_samples_per_second": 1.365,
       "train_steps_per_second": 0.042
     }
   ],
   "logging_steps": 10,
-  "max_steps": 218,
-  "num_train_epochs": 2,
+  "max_steps": 109,
+  "num_train_epochs": 1,
   "save_steps": 500,
   "total_flos": 0.0,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe8f0c39b850ed7fd27d2d9d44c44150886dfb15af5e347d3620975ca4e1e90a
+oid sha256:3ade3ff1207bb0aca50542a9f6fec0fb54178c6bb14556c5d0f48ce4a8b973c2
 size 4728