TX

README.md

@@ -0,0 +1,75 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-Instruct-v0.2
+tags:
+- alignment_handbook-handbook
+- generated_from_trainer
+datasets:
+- princeton-nlp/mistral-instruct-ultrafeedback
+model-index:
+- name: Mistral-7B-Instruct-v0.2-MI-2e-5
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/tengxiao01/huggingface/runs/a5eg2ijs)
+# Mistral-7B-Instruct-v0.2-MI-2e-5
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on the princeton-nlp/mistral-instruct-ultrafeedback dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.5846
+- Rewards/chosen: -0.8695
+- Rewards/rejected: -0.9117
+- Rewards/accuracies: 0.5559
+- Rewards/margins: 0.0422
+- Logps/rejected: -0.9117
+- Logps/chosen: -0.8695
+- Logits/rejected: -2.8388
+- Logits/chosen: -2.8404
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 1.5218        | 0.8573 | 400  | 1.5846          | -0.8695        | -0.9117          | 0.5559             | 0.0422          | -0.9117        | -0.8695      | -2.8388         | -2.8404       |
+
+
+### Framework versions
+
+- Transformers 4.42.0
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,22 @@
+{
+    "epoch": 0.9987943737441393,
+    "eval_logits/chosen": -2.8429527282714844,
+    "eval_logits/rejected": -2.840003728866577,
+    "eval_logps/chosen": -0.8574438691139221,
+    "eval_logps/rejected": -0.9005469679832458,
+    "eval_loss": 1.5836583375930786,
+    "eval_rewards/accuracies": 0.5598404407501221,
+    "eval_rewards/chosen": -0.8574438691139221,
+    "eval_rewards/margins": 0.04310305044054985,
+    "eval_rewards/rejected": -0.9005469679832458,
+    "eval_runtime": 436.1865,
+    "eval_samples": 2994,
+    "eval_samples_per_second": 6.864,
+    "eval_steps_per_second": 0.431,
+    "total_flos": 0.0,
+    "train_loss": 1.5861326811651304,
+    "train_runtime": 19371.3426,
+    "train_samples": 59720,
+    "train_samples_per_second": 3.083,
+    "train_steps_per_second": 0.024
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 0.9987943737441393,
+    "eval_logits/chosen": -2.8429527282714844,
+    "eval_logits/rejected": -2.840003728866577,
+    "eval_logps/chosen": -0.8574438691139221,
+    "eval_logps/rejected": -0.9005469679832458,
+    "eval_loss": 1.5836583375930786,
+    "eval_rewards/accuracies": 0.5598404407501221,
+    "eval_rewards/chosen": -0.8574438691139221,
+    "eval_rewards/margins": 0.04310305044054985,
+    "eval_rewards/rejected": -0.9005469679832458,
+    "eval_runtime": 436.1865,
+    "eval_samples": 2994,
+    "eval_samples_per_second": 6.864,
+    "eval_steps_per_second": 0.431
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.42.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c8b2b7b8b2ec021a0fa49ba4cd106dd3a33ff3162efc76457ca7ec94072172f
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fad0dbda4017c39a3857f6843bd52fec836701b9a5c3dfa9f6687827d342c4c9
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59585479a556dab338979199b7e08918a57a60b4c151f92263ad9bea80c84e59
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n        {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif %}\n    {%- if message['role'] == 'user' %}\n        {%- if loop.first and system_message is defined %}\n            {{- ' [INST] ' + system_message + '\\n\\n' + message['content'] + ' [/INST]' }}\n        {%- else %}\n            {{- ' [INST] ' + message['content'] + ' [/INST]' }}\n        {%- endif %}\n    {%- elif message['role'] == 'assistant' %}\n        {{- ' ' + message['content'] + eos_token}}\n    {%- else %}\n        {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n    {%- endif %}\n{%- endfor %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9987943737441393,
+    "total_flos": 0.0,
+    "train_loss": 1.5861326811651304,
+    "train_runtime": 19371.3426,
+    "train_samples": 59720,
+    "train_samples_per_second": 3.083,
+    "train_steps_per_second": 0.024
+}

trainer_state.json

@@ -0,0 +1,1453 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9987943737441393,
+  "eval_steps": 400,
+  "global_step": 466,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010716677829872739,
+      "grad_norm": 37.4633897932904,
+      "learning_rate": 2.1276595744680853e-06,
+      "logits/chosen": -2.397902250289917,
+      "logits/rejected": -2.391808271408081,
+      "logps/chosen": -0.5661157369613647,
+      "logps/rejected": -0.5547963380813599,
+      "loss": 1.5468,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.5661157369613647,
+      "rewards/margins": -0.011319375596940517,
+      "rewards/rejected": -0.5547963380813599,
+      "step": 5
+    },
+    {
+      "epoch": 0.021433355659745478,
+      "grad_norm": 18.848503699490514,
+      "learning_rate": 4.255319148936171e-06,
+      "logits/chosen": -2.374025344848633,
+      "logits/rejected": -2.345165729522705,
+      "logps/chosen": -0.5381986498832703,
+      "logps/rejected": -0.5637714266777039,
+      "loss": 1.5538,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.5381986498832703,
+      "rewards/margins": 0.025572726503014565,
+      "rewards/rejected": -0.5637714266777039,
+      "step": 10
+    },
+    {
+      "epoch": 0.032150033489618215,
+      "grad_norm": 18.865749434279863,
+      "learning_rate": 6.382978723404256e-06,
+      "logits/chosen": -2.228337526321411,
+      "logits/rejected": -2.2436676025390625,
+      "logps/chosen": -0.6078538298606873,
+      "logps/rejected": -0.6489665508270264,
+      "loss": 1.5486,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.6078538298606873,
+      "rewards/margins": 0.04111265391111374,
+      "rewards/rejected": -0.6489665508270264,
+      "step": 15
+    },
+    {
+      "epoch": 0.042866711319490956,
+      "grad_norm": 11.695416903513719,
+      "learning_rate": 8.510638297872341e-06,
+      "logits/chosen": -2.3822503089904785,
+      "logits/rejected": -2.399451732635498,
+      "logps/chosen": -0.6396132111549377,
+      "logps/rejected": -0.6679492592811584,
+      "loss": 1.5283,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.6396132111549377,
+      "rewards/margins": 0.028336137533187866,
+      "rewards/rejected": -0.6679492592811584,
+      "step": 20
+    },
+    {
+      "epoch": 0.0535833891493637,
+      "grad_norm": 8.657694670203071,
+      "learning_rate": 1.0638297872340426e-05,
+      "logits/chosen": -2.6283469200134277,
+      "logits/rejected": -2.6289353370666504,
+      "logps/chosen": -0.6763823628425598,
+      "logps/rejected": -0.7394057512283325,
+      "loss": 1.5522,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.6763823628425598,
+      "rewards/margins": 0.0630234032869339,
+      "rewards/rejected": -0.7394057512283325,
+      "step": 25
+    },
+    {
+      "epoch": 0.06430006697923643,
+      "grad_norm": 11.701639197469074,
+      "learning_rate": 1.2765957446808513e-05,
+      "logits/chosen": -2.7873454093933105,
+      "logits/rejected": -2.815898895263672,
+      "logps/chosen": -0.8007046580314636,
+      "logps/rejected": -0.8376566767692566,
+      "loss": 1.5826,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.8007046580314636,
+      "rewards/margins": 0.03695206344127655,
+      "rewards/rejected": -0.8376566767692566,
+      "step": 30
+    },
+    {
+      "epoch": 0.07501674480910918,
+      "grad_norm": 85.64689375367887,
+      "learning_rate": 1.4893617021276596e-05,
+      "logits/chosen": -2.5493829250335693,
+      "logits/rejected": -2.5297470092773438,
+      "logps/chosen": -0.9185592532157898,
+      "logps/rejected": -1.0066051483154297,
+      "loss": 1.6494,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.9185592532157898,
+      "rewards/margins": 0.08804582059383392,
+      "rewards/rejected": -1.0066051483154297,
+      "step": 35
+    },
+    {
+      "epoch": 0.08573342263898191,
+      "grad_norm": 12.618515943070657,
+      "learning_rate": 1.7021276595744682e-05,
+      "logits/chosen": -2.7925362586975098,
+      "logits/rejected": -2.7926783561706543,
+      "logps/chosen": -0.95704585313797,
+      "logps/rejected": -0.9239652752876282,
+      "loss": 1.6286,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -0.95704585313797,
+      "rewards/margins": -0.033080630004405975,
+      "rewards/rejected": -0.9239652752876282,
+      "step": 40
+    },
+    {
+      "epoch": 0.09645010046885466,
+      "grad_norm": 7.437533737509709,
+      "learning_rate": 1.914893617021277e-05,
+      "logits/chosen": -2.668349027633667,
+      "logits/rejected": -2.6576356887817383,
+      "logps/chosen": -0.9402163624763489,
+      "logps/rejected": -0.9749400019645691,
+      "loss": 1.5995,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.9402163624763489,
+      "rewards/margins": 0.034723617136478424,
+      "rewards/rejected": -0.9749400019645691,
+      "step": 45
+    },
+    {
+      "epoch": 0.1071667782987274,
+      "grad_norm": 8.099072281553982,
+      "learning_rate": 1.999747031816659e-05,
+      "logits/chosen": -2.796558380126953,
+      "logits/rejected": -2.7897918224334717,
+      "logps/chosen": -0.8397698402404785,
+      "logps/rejected": -0.8385842442512512,
+      "loss": 1.566,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.8397698402404785,
+      "rewards/margins": -0.0011855497723445296,
+      "rewards/rejected": -0.8385842442512512,
+      "step": 50
+    },
+    {
+      "epoch": 0.11788345612860013,
+      "grad_norm": 13.021189847208735,
+      "learning_rate": 1.9982015786040484e-05,
+      "logits/chosen": -2.577963352203369,
+      "logits/rejected": -2.5813965797424316,
+      "logps/chosen": -0.9185325503349304,
+      "logps/rejected": -0.9634258151054382,
+      "loss": 1.587,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.9185325503349304,
+      "rewards/margins": 0.044893164187669754,
+      "rewards/rejected": -0.9634258151054382,
+      "step": 55
+    },
+    {
+      "epoch": 0.12860013395847286,
+      "grad_norm": 29.275115325498533,
+      "learning_rate": 1.9952533792092286e-05,
+      "logits/chosen": -2.7369465827941895,
+      "logits/rejected": -2.746910810470581,
+      "logps/chosen": -0.8204312324523926,
+      "logps/rejected": -0.871277928352356,
+      "loss": 1.6101,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.8204312324523926,
+      "rewards/margins": 0.05084659531712532,
+      "rewards/rejected": -0.871277928352356,
+      "step": 60
+    },
+    {
+      "epoch": 0.13931681178834562,
+      "grad_norm": 9.013295050879966,
+      "learning_rate": 1.9909065766585522e-05,
+      "logits/chosen": -2.476846694946289,
+      "logits/rejected": -2.4859910011291504,
+      "logps/chosen": -0.8948932886123657,
+      "logps/rejected": -0.9379276037216187,
+      "loss": 1.6324,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.8948932886123657,
+      "rewards/margins": 0.043034326285123825,
+      "rewards/rejected": -0.9379276037216187,
+      "step": 65
+    },
+    {
+      "epoch": 0.15003348961821836,
+      "grad_norm": 8.308683516584036,
+      "learning_rate": 1.9851672793982618e-05,
+      "logits/chosen": -2.443450450897217,
+      "logits/rejected": -2.465125799179077,
+      "logps/chosen": -0.957764744758606,
+      "logps/rejected": -0.9942213296890259,
+      "loss": 1.6029,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.957764744758606,
+      "rewards/margins": 0.03645653277635574,
+      "rewards/rejected": -0.9942213296890259,
+      "step": 70
+    },
+    {
+      "epoch": 0.1607501674480911,
+      "grad_norm": 7.915844945947729,
+      "learning_rate": 1.978043552710454e-05,
+      "logits/chosen": -2.4849705696105957,
+      "logits/rejected": -2.4826462268829346,
+      "logps/chosen": -0.9050397872924805,
+      "logps/rejected": -0.9400200843811035,
+      "loss": 1.6052,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.9050397872924805,
+      "rewards/margins": 0.03498036414384842,
+      "rewards/rejected": -0.9400200843811035,
+      "step": 75
+    },
+    {
+      "epoch": 0.17146684527796383,
+      "grad_norm": 304.0629881690128,
+      "learning_rate": 1.9695454073791517e-05,
+      "logits/chosen": -2.500096559524536,
+      "logits/rejected": -2.5168652534484863,
+      "logps/chosen": -0.906775951385498,
+      "logps/rejected": -1.02707040309906,
+      "loss": 1.82,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.906775951385498,
+      "rewards/margins": 0.12029434740543365,
+      "rewards/rejected": -1.02707040309906,
+      "step": 80
+    },
+    {
+      "epoch": 0.18218352310783656,
+      "grad_norm": 10.960148765645817,
+      "learning_rate": 1.9596847856224143e-05,
+      "logits/chosen": -2.257906436920166,
+      "logits/rejected": -2.265631675720215,
+      "logps/chosen": -1.1177845001220703,
+      "logps/rejected": -1.1891874074935913,
+      "loss": 1.6996,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.1177845001220703,
+      "rewards/margins": 0.07140282541513443,
+      "rewards/rejected": -1.1891874074935913,
+      "step": 85
+    },
+    {
+      "epoch": 0.19290020093770932,
+      "grad_norm": 10.383486663685009,
+      "learning_rate": 1.9484755443102573e-05,
+      "logits/chosen": -2.215013027191162,
+      "logits/rejected": -2.2123208045959473,
+      "logps/chosen": -1.035982370376587,
+      "logps/rejected": -1.0804064273834229,
+      "loss": 1.6576,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.035982370376587,
+      "rewards/margins": 0.044424109160900116,
+      "rewards/rejected": -1.0804064273834229,
+      "step": 90
+    },
+    {
+      "epoch": 0.20361687876758205,
+      "grad_norm": 9.444434646488055,
+      "learning_rate": 1.9359334354919587e-05,
+      "logits/chosen": -2.5854439735412598,
+      "logits/rejected": -2.604400157928467,
+      "logps/chosen": -1.0645908117294312,
+      "logps/rejected": -1.1030633449554443,
+      "loss": 1.6255,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.0645908117294312,
+      "rewards/margins": 0.03847253695130348,
+      "rewards/rejected": -1.1030633449554443,
+      "step": 95
+    },
+    {
+      "epoch": 0.2143335565974548,
+      "grad_norm": 10.008429713827578,
+      "learning_rate": 1.9220760842601245e-05,
+      "logits/chosen": -2.41093373298645,
+      "logits/rejected": -2.416090488433838,
+      "logps/chosen": -1.0662959814071655,
+      "logps/rejected": -1.0942766666412354,
+      "loss": 1.6366,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.0662959814071655,
+      "rewards/margins": 0.027980733662843704,
+      "rewards/rejected": -1.0942766666412354,
+      "step": 100
+    },
+    {
+      "epoch": 0.22505023442732752,
+      "grad_norm": 13.793372325395232,
+      "learning_rate": 1.9069229639826113e-05,
+      "logits/chosen": -2.593897581100464,
+      "logits/rejected": -2.592832326889038,
+      "logps/chosen": -1.0412575006484985,
+      "logps/rejected": -1.1209934949874878,
+      "loss": 1.6663,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.0412575006484985,
+      "rewards/margins": 0.0797361508011818,
+      "rewards/rejected": -1.1209934949874878,
+      "step": 105
+    },
+    {
+      "epoch": 0.23576691225720026,
+      "grad_norm": 25.707371820916816,
+      "learning_rate": 1.8904953689371223e-05,
+      "logits/chosen": -2.1585533618927,
+      "logits/rejected": -2.143022060394287,
+      "logps/chosen": -1.0903128385543823,
+      "logps/rejected": -1.1769222021102905,
+      "loss": 1.6708,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.0903128385543823,
+      "rewards/margins": 0.08660950511693954,
+      "rewards/rejected": -1.1769222021102905,
+      "step": 110
+    },
+    {
+      "epoch": 0.24648359008707302,
+      "grad_norm": 15.08396540818952,
+      "learning_rate": 1.8728163843869215e-05,
+      "logits/chosen": -2.3684897422790527,
+      "logits/rejected": -2.3717474937438965,
+      "logps/chosen": -1.1836369037628174,
+      "logps/rejected": -1.1988976001739502,
+      "loss": 1.8024,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.1836369037628174,
+      "rewards/margins": 0.015260684303939342,
+      "rewards/rejected": -1.1988976001739502,
+      "step": 115
+    },
+    {
+      "epoch": 0.2572002679169457,
+      "grad_norm": 10.565734009587494,
+      "learning_rate": 1.853910854139727e-05,
+      "logits/chosen": -2.122260570526123,
+      "logits/rejected": -2.138091802597046,
+      "logps/chosen": -1.133177399635315,
+      "logps/rejected": -1.1549627780914307,
+      "loss": 1.7056,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.133177399635315,
+      "rewards/margins": 0.02178535796701908,
+      "rewards/rejected": -1.1549627780914307,
+      "step": 120
+    },
+    {
+      "epoch": 0.2679169457468185,
+      "grad_norm": 7.929268982931603,
+      "learning_rate": 1.833805345635369e-05,
+      "logits/chosen": -2.467066526412964,
+      "logits/rejected": -2.470454692840576,
+      "logps/chosen": -1.0294818878173828,
+      "logps/rejected": -1.0639444589614868,
+      "loss": 1.6829,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.0294818878173828,
+      "rewards/margins": 0.03446248173713684,
+      "rewards/rejected": -1.0639444589614868,
+      "step": 125
+    },
+    {
+      "epoch": 0.27863362357669125,
+      "grad_norm": 9.40947063841382,
+      "learning_rate": 1.8125281126112678e-05,
+      "logits/chosen": -2.5180554389953613,
+      "logits/rejected": -2.5261032581329346,
+      "logps/chosen": -1.044611930847168,
+      "logps/rejected": -1.1444398164749146,
+      "loss": 1.6283,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.044611930847168,
+      "rewards/margins": 0.09982797503471375,
+      "rewards/rejected": -1.1444398164749146,
+      "step": 130
+    },
+    {
+      "epoch": 0.289350301406564,
+      "grad_norm": 7.797174649236775,
+      "learning_rate": 1.790109055398212e-05,
+      "logits/chosen": -2.5548009872436523,
+      "logits/rejected": -2.5629782676696777,
+      "logps/chosen": -0.9959312677383423,
+      "logps/rejected": -1.0395903587341309,
+      "loss": 1.6327,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.9959312677383423,
+      "rewards/margins": 0.04365905374288559,
+      "rewards/rejected": -1.0395903587341309,
+      "step": 135
+    },
+    {
+      "epoch": 0.3000669792364367,
+      "grad_norm": 8.65910654888172,
+      "learning_rate": 1.766579678902212e-05,
+      "logits/chosen": -2.452951431274414,
+      "logits/rejected": -2.459916114807129,
+      "logps/chosen": -0.9893513917922974,
+      "logps/rejected": -1.0323150157928467,
+      "loss": 1.6058,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.9893513917922974,
+      "rewards/margins": 0.042963750660419464,
+      "rewards/rejected": -1.0323150157928467,
+      "step": 140
+    },
+    {
+      "epoch": 0.31078365706630945,
+      "grad_norm": 8.475342441948424,
+      "learning_rate": 1.741973048331496e-05,
+      "logits/chosen": -2.608076810836792,
+      "logits/rejected": -2.592602491378784,
+      "logps/chosen": -0.959572434425354,
+      "logps/rejected": -1.034380316734314,
+      "loss": 1.5962,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.959572434425354,
+      "rewards/margins": 0.07480800151824951,
+      "rewards/rejected": -1.034380316734314,
+      "step": 145
+    },
+    {
+      "epoch": 0.3215003348961822,
+      "grad_norm": 13.2413080829705,
+      "learning_rate": 1.7163237427308477e-05,
+      "logits/chosen": -2.6177635192871094,
+      "logits/rejected": -2.6168532371520996,
+      "logps/chosen": -1.004525899887085,
+      "logps/rejected": -1.0910793542861938,
+      "loss": 1.6239,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.004525899887085,
+      "rewards/margins": 0.08655332773923874,
+      "rewards/rejected": -1.0910793542861938,
+      "step": 150
+    },
+    {
+      "epoch": 0.3322170127260549,
+      "grad_norm": 8.151298227822702,
+      "learning_rate": 1.689667806388594e-05,
+      "logits/chosen": -2.7449164390563965,
+      "logits/rejected": -2.7318592071533203,
+      "logps/chosen": -0.9720630645751953,
+      "logps/rejected": -1.047321081161499,
+      "loss": 1.5985,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.9720630645751953,
+      "rewards/margins": 0.0752580463886261,
+      "rewards/rejected": -1.047321081161499,
+      "step": 155
+    },
+    {
+      "epoch": 0.34293369055592765,
+      "grad_norm": 7.660153117545113,
+      "learning_rate": 1.6620426981845233e-05,
+      "logits/chosen": -2.5920422077178955,
+      "logits/rejected": -2.5721192359924316,
+      "logps/chosen": -0.9702916145324707,
+      "logps/rejected": -1.0599411725997925,
+      "loss": 1.6107,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.9702916145324707,
+      "rewards/margins": 0.08964961767196655,
+      "rewards/rejected": -1.0599411725997925,
+      "step": 160
+    },
+    {
+      "epoch": 0.3536503683858004,
+      "grad_norm": 469.42247386416926,
+      "learning_rate": 1.633487238949915e-05,
+      "logits/chosen": -2.528369426727295,
+      "logits/rejected": -2.5209622383117676,
+      "logps/chosen": -0.9299852252006531,
+      "logps/rejected": -0.9966603517532349,
+      "loss": 1.6799,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.9299852252006531,
+      "rewards/margins": 0.06667519360780716,
+      "rewards/rejected": -0.9966603517532349,
+      "step": 165
+    },
+    {
+      "epoch": 0.3643670462156731,
+      "grad_norm": 13.123351141683822,
+      "learning_rate": 1.6040415569136586e-05,
+      "logits/chosen": -2.3123598098754883,
+      "logits/rejected": -2.3209071159362793,
+      "logps/chosen": -1.2778890132904053,
+      "logps/rejected": -1.3249436616897583,
+      "loss": 1.7418,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2778890132904053,
+      "rewards/margins": 0.04705450311303139,
+      "rewards/rejected": -1.3249436616897583,
+      "step": 170
+    },
+    {
+      "epoch": 0.3750837240455459,
+      "grad_norm": 7.9321661768835865,
+      "learning_rate": 1.5737470313111134e-05,
+      "logits/chosen": -2.1733322143554688,
+      "logits/rejected": -2.188865900039673,
+      "logps/chosen": -1.1108664274215698,
+      "logps/rejected": -1.1735206842422485,
+      "loss": 1.6869,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.1108664274215698,
+      "rewards/margins": 0.06265412271022797,
+      "rewards/rejected": -1.1735206842422485,
+      "step": 175
+    },
+    {
+      "epoch": 0.38580040187541864,
+      "grad_norm": 8.557996815044758,
+      "learning_rate": 1.5426462342349736e-05,
+      "logits/chosen": -2.3044323921203613,
+      "logits/rejected": -2.3116490840911865,
+      "logps/chosen": -1.1034280061721802,
+      "logps/rejected": -1.1603246927261353,
+      "loss": 1.6567,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.1034280061721802,
+      "rewards/margins": 0.05689678341150284,
+      "rewards/rejected": -1.1603246927261353,
+      "step": 180
+    },
+    {
+      "epoch": 0.3965170797052914,
+      "grad_norm": 10.818770194479542,
+      "learning_rate": 1.5107828708098327e-05,
+      "logits/chosen": -2.3942196369171143,
+      "logits/rejected": -2.4129745960235596,
+      "logps/chosen": -1.0460340976715088,
+      "logps/rejected": -1.105322241783142,
+      "loss": 1.6177,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.0460340976715088,
+      "rewards/margins": 0.059288132935762405,
+      "rewards/rejected": -1.105322241783142,
+      "step": 185
+    },
+    {
+      "epoch": 0.4072337575351641,
+      "grad_norm": 15.512656922129596,
+      "learning_rate": 1.4782017177745322e-05,
+      "logits/chosen": -2.4357337951660156,
+      "logits/rejected": -2.458134889602661,
+      "logps/chosen": -1.0308730602264404,
+      "logps/rejected": -1.2367624044418335,
+      "loss": 1.6917,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.0308730602264404,
+      "rewards/margins": 0.20588929951190948,
+      "rewards/rejected": -1.2367624044418335,
+      "step": 190
+    },
+    {
+      "epoch": 0.41795043536503684,
+      "grad_norm": 10.838461128892558,
+      "learning_rate": 1.444948560558601e-05,
+      "logits/chosen": -2.4340341091156006,
+      "logits/rejected": -2.4456231594085693,
+      "logps/chosen": -1.2891473770141602,
+      "logps/rejected": -1.3041666746139526,
+      "loss": 1.7583,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.2891473770141602,
+      "rewards/margins": 0.015019327402114868,
+      "rewards/rejected": -1.3041666746139526,
+      "step": 195
+    },
+    {
+      "epoch": 0.4286671131949096,
+      "grad_norm": 8.28412245267914,
+      "learning_rate": 1.4110701289412045e-05,
+      "logits/chosen": -2.515929937362671,
+      "logits/rejected": -2.5307044982910156,
+      "logps/chosen": -1.1040905714035034,
+      "logps/rejected": -1.1781259775161743,
+      "loss": 1.6734,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.1040905714035034,
+      "rewards/margins": 0.07403533160686493,
+      "rewards/rejected": -1.1781259775161743,
+      "step": 200
+    },
+    {
+      "epoch": 0.4393837910247823,
+      "grad_norm": 9.96354155003577,
+      "learning_rate": 1.3766140313830279e-05,
+      "logits/chosen": -2.531589984893799,
+      "logits/rejected": -2.520904302597046,
+      "logps/chosen": -1.0145483016967773,
+      "logps/rejected": -1.0664831399917603,
+      "loss": 1.6419,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.0145483016967773,
+      "rewards/margins": 0.05193489044904709,
+      "rewards/rejected": -1.0664831399917603,
+      "step": 205
+    },
+    {
+      "epoch": 0.45010046885465504,
+      "grad_norm": 7.434219003968733,
+      "learning_rate": 1.341628688123369e-05,
+      "logits/chosen": -2.517782211303711,
+      "logits/rejected": -2.5262608528137207,
+      "logps/chosen": -1.0025173425674438,
+      "logps/rejected": -1.0794353485107422,
+      "loss": 1.606,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.0025173425674438,
+      "rewards/margins": 0.07691796869039536,
+      "rewards/rejected": -1.0794353485107422,
+      "step": 210
+    },
+    {
+      "epoch": 0.4608171466845278,
+      "grad_norm": 7.337174471827815,
+      "learning_rate": 1.3061632631364613e-05,
+      "logits/chosen": -2.5459511280059814,
+      "logits/rejected": -2.546247959136963,
+      "logps/chosen": -0.9931095242500305,
+      "logps/rejected": -1.094226598739624,
+      "loss": 1.594,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.9931095242500305,
+      "rewards/margins": 0.10111699998378754,
+      "rewards/rejected": -1.094226598739624,
+      "step": 215
+    },
+    {
+      "epoch": 0.4715338245144005,
+      "grad_norm": 8.701123471362035,
+      "learning_rate": 1.2702675950426477e-05,
+      "logits/chosen": -2.7032644748687744,
+      "logits/rejected": -2.685612440109253,
+      "logps/chosen": -0.9561999440193176,
+      "logps/rejected": -1.0159614086151123,
+      "loss": 1.63,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.9561999440193176,
+      "rewards/margins": 0.05976147577166557,
+      "rewards/rejected": -1.0159614086151123,
+      "step": 220
+    },
+    {
+      "epoch": 0.4822505023442733,
+      "grad_norm": 10.855107466518529,
+      "learning_rate": 1.2339921270714873e-05,
+      "logits/chosen": -2.7168991565704346,
+      "logits/rejected": -2.742966651916504,
+      "logps/chosen": -0.9448912739753723,
+      "logps/rejected": -1.0768978595733643,
+      "loss": 1.5893,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.9448912739753723,
+      "rewards/margins": 0.1320066601037979,
+      "rewards/rejected": -1.0768978595733643,
+      "step": 225
+    },
+    {
+      "epoch": 0.49296718017414604,
+      "grad_norm": 7.921384206654596,
+      "learning_rate": 1.1973878361752263e-05,
+      "logits/chosen": -2.7181243896484375,
+      "logits/rejected": -2.7184972763061523,
+      "logps/chosen": -0.9301810264587402,
+      "logps/rejected": -1.000950813293457,
+      "loss": 1.5781,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.9301810264587402,
+      "rewards/margins": 0.07076982408761978,
+      "rewards/rejected": -1.000950813293457,
+      "step": 230
+    },
+    {
+      "epoch": 0.5036838580040187,
+      "grad_norm": 8.000512303019988,
+      "learning_rate": 1.160506161392239e-05,
+      "logits/chosen": -2.7740559577941895,
+      "logits/rejected": -2.767812967300415,
+      "logps/chosen": -0.906406044960022,
+      "logps/rejected": -1.006753921508789,
+      "loss": 1.5802,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.906406044960022,
+      "rewards/margins": 0.10034787654876709,
+      "rewards/rejected": -1.006753921508789,
+      "step": 235
+    },
+    {
+      "epoch": 0.5144005358338914,
+      "grad_norm": 9.398458173467265,
+      "learning_rate": 1.1233989315611132e-05,
+      "logits/chosen": -2.792450428009033,
+      "logits/rejected": -2.7874622344970703,
+      "logps/chosen": -0.9394813776016235,
+      "logps/rejected": -1.022139549255371,
+      "loss": 1.587,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.9394813776016235,
+      "rewards/margins": 0.08265805244445801,
+      "rewards/rejected": -1.022139549255371,
+      "step": 240
+    },
+    {
+      "epoch": 0.5251172136637642,
+      "grad_norm": 10.738263842024553,
+      "learning_rate": 1.0861182924869584e-05,
+      "logits/chosen": -2.721609115600586,
+      "logits/rejected": -2.712308406829834,
+      "logps/chosen": -0.9511491656303406,
+      "logps/rejected": -1.0187407732009888,
+      "loss": 1.5773,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.9511491656303406,
+      "rewards/margins": 0.06759153306484222,
+      "rewards/rejected": -1.0187407732009888,
+      "step": 245
+    },
+    {
+      "epoch": 0.535833891493637,
+      "grad_norm": 8.609658733354994,
+      "learning_rate": 1.0487166336622903e-05,
+      "logits/chosen": -2.714517116546631,
+      "logits/rejected": -2.7152135372161865,
+      "logps/chosen": -0.9705566167831421,
+      "logps/rejected": -1.0084294080734253,
+      "loss": 1.5971,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.9705566167831421,
+      "rewards/margins": 0.03787278011441231,
+      "rewards/rejected": -1.0084294080734253,
+      "step": 250
+    },
+    {
+      "epoch": 0.5465505693235098,
+      "grad_norm": 8.407135051661365,
+      "learning_rate": 1.0112465146454681e-05,
+      "logits/chosen": -2.790414333343506,
+      "logits/rejected": -2.799837112426758,
+      "logps/chosen": -0.8985916376113892,
+      "logps/rejected": -0.9925104379653931,
+      "loss": 1.5696,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8985916376113892,
+      "rewards/margins": 0.09391861408948898,
+      "rewards/rejected": -0.9925104379653931,
+      "step": 255
+    },
+    {
+      "epoch": 0.5572672471533825,
+      "grad_norm": 8.166210902327252,
+      "learning_rate": 9.737605912001413e-06,
+      "logits/chosen": -2.7387256622314453,
+      "logits/rejected": -2.7400269508361816,
+      "logps/chosen": -0.9064778089523315,
+      "logps/rejected": -1.0061310529708862,
+      "loss": 1.5824,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.9064778089523315,
+      "rewards/margins": 0.09965313225984573,
+      "rewards/rejected": -1.0061310529708862,
+      "step": 260
+    },
+    {
+      "epoch": 0.5679839249832552,
+      "grad_norm": 7.963553030396463,
+      "learning_rate": 9.363115412995038e-06,
+      "logits/chosen": -2.733085870742798,
+      "logits/rejected": -2.7209997177124023,
+      "logps/chosen": -0.9366081357002258,
+      "logps/rejected": -1.074143648147583,
+      "loss": 1.5601,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.9366081357002258,
+      "rewards/margins": 0.13753557205200195,
+      "rewards/rejected": -1.074143648147583,
+      "step": 265
+    },
+    {
+      "epoch": 0.578700602813128,
+      "grad_norm": 8.394463914962087,
+      "learning_rate": 8.989519910993357e-06,
+      "logits/chosen": -2.741532802581787,
+      "logits/rejected": -2.7389678955078125,
+      "logps/chosen": -0.8945878744125366,
+      "logps/rejected": -0.9697369337081909,
+      "loss": 1.5652,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.8945878744125366,
+      "rewards/margins": 0.07514907419681549,
+      "rewards/rejected": -0.9697369337081909,
+      "step": 270
+    },
+    {
+      "epoch": 0.5894172806430007,
+      "grad_norm": 7.2839753266908875,
+      "learning_rate": 8.617344409838631e-06,
+      "logits/chosen": -2.837791919708252,
+      "logits/rejected": -2.8426520824432373,
+      "logps/chosen": -0.8537940979003906,
+      "logps/rejected": -0.9409109354019165,
+      "loss": 1.5685,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.8537940979003906,
+      "rewards/margins": 0.08711685240268707,
+      "rewards/rejected": -0.9409109354019165,
+      "step": 275
+    },
+    {
+      "epoch": 0.6001339584728734,
+      "grad_norm": 7.994187957031587,
+      "learning_rate": 8.247111917883634e-06,
+      "logits/chosen": -2.6431148052215576,
+      "logits/rejected": -2.644808530807495,
+      "logps/chosen": -0.8992069959640503,
+      "logps/rejected": -1.0336244106292725,
+      "loss": 1.5658,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.8992069959640503,
+      "rewards/margins": 0.1344173550605774,
+      "rewards/rejected": -1.0336244106292725,
+      "step": 280
+    },
+    {
+      "epoch": 0.6108506363027462,
+      "grad_norm": 10.3254576486645,
+      "learning_rate": 7.879342713021875e-06,
+      "logits/chosen": -2.745375156402588,
+      "logits/rejected": -2.744263172149658,
+      "logps/chosen": -0.9114044308662415,
+      "logps/rejected": -0.9392461776733398,
+      "loss": 1.5753,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.9114044308662415,
+      "rewards/margins": 0.027841681614518166,
+      "rewards/rejected": -0.9392461776733398,
+      "step": 285
+    },
+    {
+      "epoch": 0.6215673141326189,
+      "grad_norm": 14.256488881900221,
+      "learning_rate": 7.514553611554909e-06,
+      "logits/chosen": -2.9054572582244873,
+      "logits/rejected": -2.915325880050659,
+      "logps/chosen": -0.8290088772773743,
+      "logps/rejected": -0.9459854364395142,
+      "loss": 1.5614,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.8290088772773743,
+      "rewards/margins": 0.11697645485401154,
+      "rewards/rejected": -0.9459854364395142,
+      "step": 290
+    },
+    {
+      "epoch": 0.6322839919624916,
+      "grad_norm": 8.049072856223585,
+      "learning_rate": 7.15325724192409e-06,
+      "logits/chosen": -2.861290454864502,
+      "logits/rejected": -2.8629579544067383,
+      "logps/chosen": -0.8957312703132629,
+      "logps/rejected": -1.0799039602279663,
+      "loss": 1.5436,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8957312703132629,
+      "rewards/margins": 0.1841726452112198,
+      "rewards/rejected": -1.0799039602279663,
+      "step": 295
+    },
+    {
+      "epoch": 0.6430006697923644,
+      "grad_norm": 8.353215812366958,
+      "learning_rate": 6.79596132432742e-06,
+      "logits/chosen": -2.8521814346313477,
+      "logits/rejected": -2.8686671257019043,
+      "logps/chosen": -0.8559802770614624,
+      "logps/rejected": -0.9652966260910034,
+      "loss": 1.5273,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8559802770614624,
+      "rewards/margins": 0.10931621491909027,
+      "rewards/rejected": -0.9652966260910034,
+      "step": 300
+    },
+    {
+      "epoch": 0.6537173476222371,
+      "grad_norm": 8.558746762469774,
+      "learning_rate": 6.443167957233864e-06,
+      "logits/chosen": -2.8060519695281982,
+      "logits/rejected": -2.793569326400757,
+      "logps/chosen": -0.8676208257675171,
+      "logps/rejected": -0.9558470845222473,
+      "loss": 1.5333,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8676208257675171,
+      "rewards/margins": 0.08822615444660187,
+      "rewards/rejected": -0.9558470845222473,
+      "step": 305
+    },
+    {
+      "epoch": 0.6644340254521098,
+      "grad_norm": 8.944155497280653,
+      "learning_rate": 6.095372911797673e-06,
+      "logits/chosen": -2.8632490634918213,
+      "logits/rejected": -2.859429359436035,
+      "logps/chosen": -0.8489475250244141,
+      "logps/rejected": -0.9311763644218445,
+      "loss": 1.5397,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.8489475250244141,
+      "rewards/margins": 0.08222876489162445,
+      "rewards/rejected": -0.9311763644218445,
+      "step": 310
+    },
+    {
+      "epoch": 0.6751507032819826,
+      "grad_norm": 7.822389173887281,
+      "learning_rate": 5.753064935164348e-06,
+      "logits/chosen": -2.7866902351379395,
+      "logits/rejected": -2.792357921600342,
+      "logps/chosen": -0.84296715259552,
+      "logps/rejected": -0.959184467792511,
+      "loss": 1.5381,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.84296715259552,
+      "rewards/margins": 0.11621735244989395,
+      "rewards/rejected": -0.959184467792511,
+      "step": 315
+    },
+    {
+      "epoch": 0.6858673811118553,
+      "grad_norm": 9.32294397022148,
+      "learning_rate": 5.4167250636472426e-06,
+      "logits/chosen": -2.7923617362976074,
+      "logits/rejected": -2.7717807292938232,
+      "logps/chosen": -0.8463284373283386,
+      "logps/rejected": -0.9291221499443054,
+      "loss": 1.5458,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.8463284373283386,
+      "rewards/margins": 0.08279372751712799,
+      "rewards/rejected": -0.9291221499443054,
+      "step": 320
+    },
+    {
+      "epoch": 0.696584058941728,
+      "grad_norm": 8.808118982464391,
+      "learning_rate": 5.08682594674002e-06,
+      "logits/chosen": -2.7373509407043457,
+      "logits/rejected": -2.7527077198028564,
+      "logps/chosen": -0.8056324124336243,
+      "logps/rejected": -0.957785964012146,
+      "loss": 1.5196,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.8056324124336243,
+      "rewards/margins": 0.15215365588665009,
+      "rewards/rejected": -0.957785964012146,
+      "step": 325
+    },
+    {
+      "epoch": 0.7073007367716008,
+      "grad_norm": 8.169267761293227,
+      "learning_rate": 4.763831182914848e-06,
+      "logits/chosen": -2.7620842456817627,
+      "logits/rejected": -2.77044415473938,
+      "logps/chosen": -0.8720673322677612,
+      "logps/rejected": -0.9474166631698608,
+      "loss": 1.5373,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.8720673322677612,
+      "rewards/margins": 0.07534926384687424,
+      "rewards/rejected": -0.9474166631698608,
+      "step": 330
+    },
+    {
+      "epoch": 0.7180174146014735,
+      "grad_norm": 8.7345666789259,
+      "learning_rate": 4.448194668139777e-06,
+      "logits/chosen": -2.794130325317383,
+      "logits/rejected": -2.7945706844329834,
+      "logps/chosen": -0.8377755284309387,
+      "logps/rejected": -0.9672821760177612,
+      "loss": 1.5233,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.8377755284309387,
+      "rewards/margins": 0.12950672209262848,
+      "rewards/rejected": -0.9672821760177612,
+      "step": 335
+    },
+    {
+      "epoch": 0.7287340924313462,
+      "grad_norm": 8.389766148627231,
+      "learning_rate": 4.1403599580307795e-06,
+      "logits/chosen": -2.714247226715088,
+      "logits/rejected": -2.7236764430999756,
+      "logps/chosen": -0.8354495167732239,
+      "logps/rejected": -0.9634197354316711,
+      "loss": 1.5335,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.8354495167732239,
+      "rewards/margins": 0.1279701441526413,
+      "rewards/rejected": -0.9634197354316711,
+      "step": 340
+    },
+    {
+      "epoch": 0.739450770261219,
+      "grad_norm": 9.147040621292957,
+      "learning_rate": 3.840759644534828e-06,
+      "logits/chosen": -2.790487766265869,
+      "logits/rejected": -2.8023035526275635,
+      "logps/chosen": -0.8659473657608032,
+      "logps/rejected": -0.9856260418891907,
+      "loss": 1.5348,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.8659473657608032,
+      "rewards/margins": 0.11967878043651581,
+      "rewards/rejected": -0.9856260418891907,
+      "step": 345
+    },
+    {
+      "epoch": 0.7501674480910918,
+      "grad_norm": 8.599069402922064,
+      "learning_rate": 3.549814748019906e-06,
+      "logits/chosen": -2.815507411956787,
+      "logits/rejected": -2.824169158935547,
+      "logps/chosen": -0.808633029460907,
+      "logps/rejected": -0.9542997479438782,
+      "loss": 1.529,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.808633029460907,
+      "rewards/margins": 0.14566662907600403,
+      "rewards/rejected": -0.9542997479438782,
+      "step": 350
+    },
+    {
+      "epoch": 0.7608841259209645,
+      "grad_norm": 7.982319756803084,
+      "learning_rate": 3.2679341256262653e-06,
+      "logits/chosen": -2.8472940921783447,
+      "logits/rejected": -2.8238396644592285,
+      "logps/chosen": -0.9044294357299805,
+      "logps/rejected": -0.9391825795173645,
+      "loss": 1.5418,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.9044294357299805,
+      "rewards/margins": 0.03475295007228851,
+      "rewards/rejected": -0.9391825795173645,
+      "step": 355
+    },
+    {
+      "epoch": 0.7716008037508373,
+      "grad_norm": 11.32319755235924,
+      "learning_rate": 2.995513896710327e-06,
+      "logits/chosen": -2.8410215377807617,
+      "logits/rejected": -2.83413028717041,
+      "logps/chosen": -0.7960796356201172,
+      "logps/rejected": -0.9083698391914368,
+      "loss": 1.5306,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.7960796356201172,
+      "rewards/margins": 0.1122901439666748,
+      "rewards/rejected": -0.9083698391914368,
+      "step": 360
+    },
+    {
+      "epoch": 0.78231748158071,
+      "grad_norm": 7.8205965789155325,
+      "learning_rate": 2.7329368861886862e-06,
+      "logits/chosen": -2.8469526767730713,
+      "logits/rejected": -2.8535444736480713,
+      "logps/chosen": -0.8426831960678101,
+      "logps/rejected": -0.9201619029045105,
+      "loss": 1.5523,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.8426831960678101,
+      "rewards/margins": 0.07747860997915268,
+      "rewards/rejected": -0.9201619029045105,
+      "step": 365
+    },
+    {
+      "epoch": 0.7930341594105828,
+      "grad_norm": 11.791063151570222,
+      "learning_rate": 2.4805720865643943e-06,
+      "logits/chosen": -2.8243472576141357,
+      "logits/rejected": -2.8304176330566406,
+      "logps/chosen": -0.8739852905273438,
+      "logps/rejected": -0.9622358083724976,
+      "loss": 1.5445,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.8739852905273438,
+      "rewards/margins": 0.08825065940618515,
+      "rewards/rejected": -0.9622358083724976,
+      "step": 370
+    },
+    {
+      "epoch": 0.8037508372404555,
+      "grad_norm": 10.592008554970972,
+      "learning_rate": 2.2387741393916117e-06,
+      "logits/chosen": -2.7737154960632324,
+      "logits/rejected": -2.7636284828186035,
+      "logps/chosen": -0.9277487993240356,
+      "logps/rejected": -1.0097343921661377,
+      "loss": 1.5368,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.9277487993240356,
+      "rewards/margins": 0.08198549598455429,
+      "rewards/rejected": -1.0097343921661377,
+      "step": 375
+    },
+    {
+      "epoch": 0.8144675150703282,
+      "grad_norm": 8.911271573494423,
+      "learning_rate": 2.0078828369072323e-06,
+      "logits/chosen": -2.7896554470062256,
+      "logits/rejected": -2.770059823989868,
+      "logps/chosen": -0.8297151327133179,
+      "logps/rejected": -0.9309379458427429,
+      "loss": 1.531,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.8297151327133179,
+      "rewards/margins": 0.10122283548116684,
+      "rewards/rejected": -0.9309379458427429,
+      "step": 380
+    },
+    {
+      "epoch": 0.825184192900201,
+      "grad_norm": 10.364577364972547,
+      "learning_rate": 1.7882226445298656e-06,
+      "logits/chosen": -2.740999460220337,
+      "logits/rejected": -2.733097791671753,
+      "logps/chosen": -0.8539925813674927,
+      "logps/rejected": -0.9543533325195312,
+      "loss": 1.5174,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8539925813674927,
+      "rewards/margins": 0.10036077350378036,
+      "rewards/rejected": -0.9543533325195312,
+      "step": 385
+    },
+    {
+      "epoch": 0.8359008707300737,
+      "grad_norm": 10.862441952775951,
+      "learning_rate": 1.580102244897217e-06,
+      "logits/chosen": -2.7668397426605225,
+      "logits/rejected": -2.7470200061798096,
+      "logps/chosen": -0.8713840246200562,
+      "logps/rejected": -0.940260112285614,
+      "loss": 1.5632,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.8713840246200562,
+      "rewards/margins": 0.0688760057091713,
+      "rewards/rejected": -0.940260112285614,
+      "step": 390
+    },
+    {
+      "epoch": 0.8466175485599464,
+      "grad_norm": 7.998296949983092,
+      "learning_rate": 1.3838141040825536e-06,
+      "logits/chosen": -2.85998272895813,
+      "logits/rejected": -2.8610522747039795,
+      "logps/chosen": -0.8187233805656433,
+      "logps/rejected": -0.8954393267631531,
+      "loss": 1.5253,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.8187233805656433,
+      "rewards/margins": 0.0767158791422844,
+      "rewards/rejected": -0.8954393267631531,
+      "step": 395
+    },
+    {
+      "epoch": 0.8573342263898192,
+      "grad_norm": 10.009174586154066,
+      "learning_rate": 1.1996340605999058e-06,
+      "logits/chosen": -2.842839479446411,
+      "logits/rejected": -2.819075107574463,
+      "logps/chosen": -0.7899017333984375,
+      "logps/rejected": -0.9768079519271851,
+      "loss": 1.5218,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.7899017333984375,
+      "rewards/margins": 0.18690629303455353,
+      "rewards/rejected": -0.9768079519271851,
+      "step": 400
+    },
+    {
+      "epoch": 0.8573342263898192,
+      "eval_logits/chosen": -2.840418815612793,
+      "eval_logits/rejected": -2.8387703895568848,
+      "eval_logps/chosen": -0.8695030808448792,
+      "eval_logps/rejected": -0.9116575121879578,
+      "eval_loss": 1.5846024751663208,
+      "eval_rewards/accuracies": 0.5558510422706604,
+      "eval_rewards/chosen": -0.8695030808448792,
+      "eval_rewards/margins": 0.0421544648706913,
+      "eval_rewards/rejected": -0.9116575121879578,
+      "eval_runtime": 432.5597,
+      "eval_samples_per_second": 6.922,
+      "eval_steps_per_second": 0.435,
+      "step": 400
+    },
+    {
+      "epoch": 0.8680509042196919,
+      "grad_norm": 8.778213042419416,
+      "learning_rate": 1.0278209377755099e-06,
+      "logits/chosen": -2.829361915588379,
+      "logits/rejected": -2.8452582359313965,
+      "logps/chosen": -0.8199126124382019,
+      "logps/rejected": -0.9467391967773438,
+      "loss": 1.5156,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.8199126124382019,
+      "rewards/margins": 0.12682652473449707,
+      "rewards/rejected": -0.9467391967773438,
+      "step": 405
+    },
+    {
+      "epoch": 0.8787675820495646,
+      "grad_norm": 8.779487119016105,
+      "learning_rate": 8.68616180030275e-07,
+      "logits/chosen": -2.865104913711548,
+      "logits/rejected": -2.867384672164917,
+      "logps/chosen": -0.8341816067695618,
+      "logps/rejected": -0.8998939394950867,
+      "loss": 1.5114,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.8341816067695618,
+      "rewards/margins": 0.06571228057146072,
+      "rewards/rejected": -0.8998939394950867,
+      "step": 410
+    },
+    {
+      "epoch": 0.8894842598794374,
+      "grad_norm": 8.92068989780619,
+      "learning_rate": 7.22243513584342e-07,
+      "logits/chosen": -2.77596378326416,
+      "logits/rejected": -2.7669265270233154,
+      "logps/chosen": -0.8306072354316711,
+      "logps/rejected": -0.9651764035224915,
+      "loss": 1.5034,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8306072354316711,
+      "rewards/margins": 0.13456913828849792,
+      "rewards/rejected": -0.9651764035224915,
+      "step": 415
+    },
+    {
+      "epoch": 0.9002009377093101,
+      "grad_norm": 9.219885917035732,
+      "learning_rate": 5.889086320605575e-07,
+      "logits/chosen": -2.7921142578125,
+      "logits/rejected": -2.7921335697174072,
+      "logps/chosen": -0.7846695184707642,
+      "logps/rejected": -0.8718813061714172,
+      "loss": 1.5351,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.7846695184707642,
+      "rewards/margins": 0.08721177279949188,
+      "rewards/rejected": -0.8718813061714172,
+      "step": 420
+    },
+    {
+      "epoch": 0.9109176155391828,
+      "grad_norm": 9.448964270757202,
+      "learning_rate": 4.6879890742869584e-07,
+      "logits/chosen": -2.7913196086883545,
+      "logits/rejected": -2.789302110671997,
+      "logps/chosen": -0.7865716218948364,
+      "logps/rejected": -0.9528288841247559,
+      "loss": 1.5108,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7865716218948364,
+      "rewards/margins": 0.16625720262527466,
+      "rewards/rejected": -0.9528288841247559,
+      "step": 425
+    },
+    {
+      "epoch": 0.9216342933690556,
+      "grad_norm": 10.369844004542797,
+      "learning_rate": 3.6208312669659405e-07,
+      "logits/chosen": -2.817171812057495,
+      "logits/rejected": -2.8241400718688965,
+      "logps/chosen": -0.8919852375984192,
+      "logps/rejected": -1.037209153175354,
+      "loss": 1.5259,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.8919852375984192,
+      "rewards/margins": 0.14522376656532288,
+      "rewards/rejected": -1.037209153175354,
+      "step": 430
+    },
+    {
+      "epoch": 0.9323509711989283,
+      "grad_norm": 12.055758939525129,
+      "learning_rate": 2.689112547182593e-07,
+      "logits/chosen": -2.890130043029785,
+      "logits/rejected": -2.882509708404541,
+      "logps/chosen": -0.7903082370758057,
+      "logps/rejected": -0.9467129707336426,
+      "loss": 1.5203,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.7903082370758057,
+      "rewards/margins": 0.15640470385551453,
+      "rewards/rejected": -0.9467129707336426,
+      "step": 435
+    },
+    {
+      "epoch": 0.943067649028801,
+      "grad_norm": 9.50477372459136,
+      "learning_rate": 1.8941422345223448e-07,
+      "logits/chosen": -2.7733447551727295,
+      "logits/rejected": -2.77803635597229,
+      "logps/chosen": -0.7930269241333008,
+      "logps/rejected": -0.9639067649841309,
+      "loss": 1.4896,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.7930269241333008,
+      "rewards/margins": 0.17087996006011963,
+      "rewards/rejected": -0.9639067649841309,
+      "step": 440
+    },
+    {
+      "epoch": 0.9537843268586738,
+      "grad_norm": 10.00023586076567,
+      "learning_rate": 1.2370374796639383e-07,
+      "logits/chosen": -2.794691324234009,
+      "logits/rejected": -2.799956798553467,
+      "logps/chosen": -0.8402734994888306,
+      "logps/rejected": -0.9617452621459961,
+      "loss": 1.5232,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8402734994888306,
+      "rewards/margins": 0.12147176265716553,
+      "rewards/rejected": -0.9617452621459961,
+      "step": 445
+    },
+    {
+      "epoch": 0.9645010046885466,
+      "grad_norm": 9.634395288787493,
+      "learning_rate": 7.18721694477209e-08,
+      "logits/chosen": -2.8351221084594727,
+      "logits/rejected": -2.833106756210327,
+      "logps/chosen": -0.8035479784011841,
+      "logps/rejected": -0.9219916462898254,
+      "loss": 1.5027,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.8035479784011841,
+      "rewards/margins": 0.11844360828399658,
+      "rewards/rejected": -0.9219916462898254,
+      "step": 450
+    },
+    {
+      "epoch": 0.9752176825184193,
+      "grad_norm": 9.629343705982949,
+      "learning_rate": 3.399232543769504e-08,
+      "logits/chosen": -2.807852268218994,
+      "logits/rejected": -2.815563678741455,
+      "logps/chosen": -0.8062378168106079,
+      "logps/rejected": -0.9637410044670105,
+      "loss": 1.5073,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.8062378168106079,
+      "rewards/margins": 0.15750321745872498,
+      "rewards/rejected": -0.9637410044670105,
+      "step": 455
+    },
+    {
+      "epoch": 0.9859343603482921,
+      "grad_norm": 10.887896034397038,
+      "learning_rate": 1.011744747562271e-08,
+      "logits/chosen": -2.826111316680908,
+      "logits/rejected": -2.830981731414795,
+      "logps/chosen": -0.8098823428153992,
+      "logps/rejected": -0.9950210452079773,
+      "loss": 1.5206,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.8098823428153992,
+      "rewards/margins": 0.1851385533809662,
+      "rewards/rejected": -0.9950210452079773,
+      "step": 460
+    },
+    {
+      "epoch": 0.9966510381781648,
+      "grad_norm": 10.914112703731178,
+      "learning_rate": 2.810862937696612e-10,
+      "logits/chosen": -2.813309907913208,
+      "logits/rejected": -2.8205251693725586,
+      "logps/chosen": -0.8651328086853027,
+      "logps/rejected": -0.9807795286178589,
+      "loss": 1.5128,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8651328086853027,
+      "rewards/margins": 0.11564669758081436,
+      "rewards/rejected": -0.9807795286178589,
+      "step": 465
+    },
+    {
+      "epoch": 0.9987943737441393,
+      "step": 466,
+      "total_flos": 0.0,
+      "train_loss": 1.5861326811651304,
+      "train_runtime": 19371.3426,
+      "train_samples_per_second": 3.083,
+      "train_steps_per_second": 0.024
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 466,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11c0a462df5fb5095b7ae3762486aa55f82a90386089e594dcc87eea91dedd2e
+size 7416