Upload folder using huggingface_hub

README.md

@@ -0,0 +1,60 @@
+---
+license: mit
+datasets:
+- HuggingFaceH4/ultrachat_200k
+language:
+- en
+pipeline_tag: text-generation
+---
+Self-Play Fine-Tuning Converts Weak Language Models to Strong Language Models (https://arxiv.org/abs/2401.01335)
+
+# zephyr-7b-sft-full-spin-iter3
+
+This model is a self-play fine-tuned model at iteration 3 from [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) using synthetic data based on on the [HuggingFaceH4/ultrachat_200k](https://huggingface.co/datasets/HuggingFaceH4/ultrachat_200k) dataset.
+
+## Model Details
+
+### Model Description
+
+- Model type: A 7B parameter GPT-like model fine-tuned on synthetic datasets.
+- Language(s) (NLP): Primarily English
+- License: MIT
+- Finetuned from model: alignment-handbook/zephyr-7b-sft-full (based on mistralai/Mistral-7B-v0.1)
+
+### Training hyperparameters
+The following hyperparameters were used during training:
+
+- learning_rate: 1e-07
+- train_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 64
+- optimizer: RMSProp 
+- lr_scheduler_type: linear 
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 2.0
+
+## [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_UCLA-AGI__test_final)
+| Metric                | Value                     |
+|-----------------------|---------------------------|
+| Avg.                  | 63.70   |
+| ARC (25-shot)         | 66.13          |
+| HellaSwag (10-shot)   | 85.85    |
+| MMLU (5-shot)         | 61.51         |
+| TruthfulQA (0-shot)   | 57.89   |
+| Winogrande (5-shot)   | 76.64   |
+| GSM8K (5-shot)        | 34.19        |
+  
+## Citation
+```
+@misc{chen2024selfplay,
+      title={Self-Play Fine-Tuning Converts Weak Language Models to Strong Language Models}, 
+      author={Zixiang Chen and Yihe Deng and Huizhuo Yuan and Kaixuan Ji and Quanquan Gu},
+      year={2024},
+      eprint={2401.01335},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+```

config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "checkpoint-3112",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.35.0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.35.0"
+}

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

output.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7764f7f582a96325d9e43f5401f413850feb949bcfe1de87a68b3058ac3e68aa
+size 3861294616

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,41 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}

trainer_state.json

@@ -0,0 +1,2443 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 1556,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.0706638115631692e-10,
+      "logits/chosen": 3.009334087371826,
+      "logits/rejected": 2.8860254287719727,
+      "logps/chosen": -363.7457580566406,
+      "logps/rejected": -405.4437255859375,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 1.070663811563169e-09,
+      "logits/chosen": 2.383711338043213,
+      "logits/rejected": 3.0388379096984863,
+      "logps/chosen": -456.6111145019531,
+      "logps/rejected": -428.7125244140625,
+      "loss": 2.7789,
+      "rewards/accuracies": 0.4305555522441864,
+      "rewards/chosen": -0.10625362396240234,
+      "rewards/margins": -0.11497735977172852,
+      "rewards/rejected": 0.008723735809326172,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.141327623126338e-09,
+      "logits/chosen": 2.6908938884735107,
+      "logits/rejected": 3.1237950325012207,
+      "logps/chosen": -387.3503112792969,
+      "logps/rejected": -384.4349060058594,
+      "loss": 2.7716,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.43295717239379883,
+      "rewards/margins": 1.0383365154266357,
+      "rewards/rejected": -0.6053793430328369,
+      "step": 20
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 3.2119914346895075e-09,
+      "logits/chosen": 2.479438543319702,
+      "logits/rejected": 2.9864134788513184,
+      "logps/chosen": -409.71563720703125,
+      "logps/rejected": -410.36260986328125,
+      "loss": 3.0305,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.3105888366699219,
+      "rewards/margins": 0.5438008308410645,
+      "rewards/rejected": -0.23321199417114258,
+      "step": 30
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 4.282655246252676e-09,
+      "logits/chosen": 2.513826847076416,
+      "logits/rejected": 3.7642135620117188,
+      "logps/chosen": -412.05780029296875,
+      "logps/rejected": -408.3034973144531,
+      "loss": 2.3915,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.4330732822418213,
+      "rewards/margins": 0.3383500576019287,
+      "rewards/rejected": -0.77142333984375,
+      "step": 40
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 5.353319057815846e-09,
+      "logits/chosen": 2.6957950592041016,
+      "logits/rejected": 3.350551128387451,
+      "logps/chosen": -428.7166442871094,
+      "logps/rejected": -425.9712829589844,
+      "loss": 2.6906,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.6460269093513489,
+      "rewards/margins": 0.9008991122245789,
+      "rewards/rejected": -0.25487232208251953,
+      "step": 50
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 6.423982869379015e-09,
+      "logits/chosen": 1.877236008644104,
+      "logits/rejected": 3.345506191253662,
+      "logps/chosen": -423.69549560546875,
+      "logps/rejected": -409.8067321777344,
+      "loss": 1.8376,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.4741673469543457,
+      "rewards/margins": 4.329440116882324,
+      "rewards/rejected": -1.8552722930908203,
+      "step": 60
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 7.494646680942184e-09,
+      "logits/chosen": 2.3659510612487793,
+      "logits/rejected": 3.347115993499756,
+      "logps/chosen": -419.614990234375,
+      "logps/rejected": -404.15386962890625,
+      "loss": 1.8117,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 1.2335608005523682,
+      "rewards/margins": 1.696711778640747,
+      "rewards/rejected": -0.4631509780883789,
+      "step": 70
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 8.565310492505352e-09,
+      "logits/chosen": 1.8567355871200562,
+      "logits/rejected": 3.529881238937378,
+      "logps/chosen": -454.572998046875,
+      "logps/rejected": -433.12152099609375,
+      "loss": 1.4964,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 1.8937450647354126,
+      "rewards/margins": 4.073466777801514,
+      "rewards/rejected": -2.1797218322753906,
+      "step": 80
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.635974304068522e-09,
+      "logits/chosen": 2.379859447479248,
+      "logits/rejected": 3.260948657989502,
+      "logps/chosen": -451.9266052246094,
+      "logps/rejected": -383.00042724609375,
+      "loss": 1.6543,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 2.33457612991333,
+      "rewards/margins": 3.6442272663116455,
+      "rewards/rejected": -1.3096508979797363,
+      "step": 90
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.0706638115631692e-08,
+      "logits/chosen": 2.4745795726776123,
+      "logits/rejected": 3.668903350830078,
+      "logps/chosen": -402.97393798828125,
+      "logps/rejected": -378.31890869140625,
+      "loss": 1.5408,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.7632139921188354,
+      "rewards/margins": 2.8585281372070312,
+      "rewards/rejected": -2.0953145027160645,
+      "step": 100
+    },
+    {
+      "epoch": 0.06,
+      "eval_logits/chosen": 1.7967612743377686,
+      "eval_logits/rejected": 2.483816146850586,
+      "eval_logps/chosen": -403.54827880859375,
+      "eval_logps/rejected": -402.92413330078125,
+      "eval_loss": 1.0591531991958618,
+      "eval_rewards/accuracies": 0.8125,
+      "eval_rewards/chosen": 2.333465814590454,
+      "eval_rewards/margins": 5.841272830963135,
+      "eval_rewards/rejected": -3.5078072547912598,
+      "eval_runtime": 76.9287,
+      "eval_samples_per_second": 12.999,
+      "eval_steps_per_second": 0.416,
+      "step": 100
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.177730192719486e-08,
+      "logits/chosen": 2.7680611610412598,
+      "logits/rejected": 3.0494232177734375,
+      "logps/chosen": -384.3839111328125,
+      "logps/rejected": -407.72918701171875,
+      "loss": 1.0018,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.2038276195526123,
+      "rewards/margins": 6.380339622497559,
+      "rewards/rejected": -4.176511764526367,
+      "step": 110
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.284796573875803e-08,
+      "logits/chosen": 2.498812198638916,
+      "logits/rejected": 3.4484715461730957,
+      "logps/chosen": -401.28857421875,
+      "logps/rejected": -426.8399963378906,
+      "loss": 0.7845,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 2.2210142612457275,
+      "rewards/margins": 8.185151100158691,
+      "rewards/rejected": -5.964136123657227,
+      "step": 120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.3918629550321198e-08,
+      "logits/chosen": 2.083467483520508,
+      "logits/rejected": 2.998776435852051,
+      "logps/chosen": -437.83709716796875,
+      "logps/rejected": -462.96026611328125,
+      "loss": 0.9019,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 4.379509449005127,
+      "rewards/margins": 10.292404174804688,
+      "rewards/rejected": -5.912895202636719,
+      "step": 130
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.4989293361884368e-08,
+      "logits/chosen": 2.5522398948669434,
+      "logits/rejected": 3.3716235160827637,
+      "logps/chosen": -422.5474548339844,
+      "logps/rejected": -406.94183349609375,
+      "loss": 0.8578,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 2.729921579360962,
+      "rewards/margins": 7.9893903732299805,
+      "rewards/rejected": -5.259469509124756,
+      "step": 140
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.6059957173447535e-08,
+      "logits/chosen": 2.7751362323760986,
+      "logits/rejected": 2.9003946781158447,
+      "logps/chosen": -391.8503723144531,
+      "logps/rejected": -400.2032470703125,
+      "loss": 0.7248,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 1.3115911483764648,
+      "rewards/margins": 8.482717514038086,
+      "rewards/rejected": -7.171125888824463,
+      "step": 150
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.7130620985010704e-08,
+      "logits/chosen": 2.3851382732391357,
+      "logits/rejected": 3.175081968307495,
+      "logps/chosen": -482.95062255859375,
+      "logps/rejected": -422.1293029785156,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 3.3662681579589844,
+      "rewards/margins": 10.24647331237793,
+      "rewards/rejected": -6.8802056312561035,
+      "step": 160
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.8201284796573874e-08,
+      "logits/chosen": 2.398458957672119,
+      "logits/rejected": 3.4337353706359863,
+      "logps/chosen": -422.200927734375,
+      "logps/rejected": -402.37274169921875,
+      "loss": 0.8347,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.8539754748344421,
+      "rewards/margins": 8.811124801635742,
+      "rewards/rejected": -7.957149505615234,
+      "step": 170
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9271948608137044e-08,
+      "logits/chosen": 2.4511101245880127,
+      "logits/rejected": 3.5071682929992676,
+      "logps/chosen": -444.2669372558594,
+      "logps/rejected": -460.7818908691406,
+      "loss": 0.7348,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.6959193348884583,
+      "rewards/margins": 10.471872329711914,
+      "rewards/rejected": -9.77595329284668,
+      "step": 180
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 2.0342612419700214e-08,
+      "logits/chosen": 2.3941550254821777,
+      "logits/rejected": 3.183414936065674,
+      "logps/chosen": -406.2020263671875,
+      "logps/rejected": -389.969970703125,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.565895140171051,
+      "rewards/margins": 9.187482833862305,
+      "rewards/rejected": -8.621587753295898,
+      "step": 190
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 2.1413276231263384e-08,
+      "logits/chosen": 2.699617624282837,
+      "logits/rejected": 3.2629177570343018,
+      "logps/chosen": -370.5373840332031,
+      "logps/rejected": -373.650634765625,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.4384472370147705,
+      "rewards/margins": 9.281099319458008,
+      "rewards/rejected": -8.8426513671875,
+      "step": 200
+    },
+    {
+      "epoch": 0.13,
+      "eval_logits/chosen": 1.8075158596038818,
+      "eval_logits/rejected": 2.4975709915161133,
+      "eval_logps/chosen": -403.94219970703125,
+      "eval_logps/rejected": -404.73687744140625,
+      "eval_loss": 0.7941137552261353,
+      "eval_rewards/accuracies": 0.9140625,
+      "eval_rewards/chosen": 0.36384421586990356,
+      "eval_rewards/margins": 12.935296058654785,
+      "eval_rewards/rejected": -12.571451187133789,
+      "eval_runtime": 76.7386,
+      "eval_samples_per_second": 13.031,
+      "eval_steps_per_second": 0.417,
+      "step": 200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 2.248394004282655e-08,
+      "logits/chosen": 2.466681718826294,
+      "logits/rejected": 3.0487618446350098,
+      "logps/chosen": -413.58941650390625,
+      "logps/rejected": -422.9561462402344,
+      "loss": 0.5453,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.9145727157592773,
+      "rewards/margins": 14.975469589233398,
+      "rewards/rejected": -13.06089973449707,
+      "step": 210
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 2.355460385438972e-08,
+      "logits/chosen": 2.4473984241485596,
+      "logits/rejected": 3.8849105834960938,
+      "logps/chosen": -435.60198974609375,
+      "logps/rejected": -394.08990478515625,
+      "loss": 0.4307,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 1.1083576679229736,
+      "rewards/margins": 12.617959976196289,
+      "rewards/rejected": -11.509601593017578,
+      "step": 220
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 2.462526766595289e-08,
+      "logits/chosen": 3.0809953212738037,
+      "logits/rejected": 3.275381088256836,
+      "logps/chosen": -387.32427978515625,
+      "logps/rejected": -418.32745361328125,
+      "loss": 0.4707,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 2.789315700531006,
+      "rewards/margins": 15.172778129577637,
+      "rewards/rejected": -12.383462905883789,
+      "step": 230
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 2.569593147751606e-08,
+      "logits/chosen": 2.8269176483154297,
+      "logits/rejected": 3.720735549926758,
+      "logps/chosen": -400.30743408203125,
+      "logps/rejected": -414.60430908203125,
+      "loss": 0.4476,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.7549598217010498,
+      "rewards/margins": 15.389744758605957,
+      "rewards/rejected": -13.634783744812012,
+      "step": 240
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 2.676659528907923e-08,
+      "logits/chosen": 2.572218894958496,
+      "logits/rejected": 2.84545636177063,
+      "logps/chosen": -398.96356201171875,
+      "logps/rejected": -404.20635986328125,
+      "loss": 0.3304,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.083133578300476,
+      "rewards/margins": 14.877154350280762,
+      "rewards/rejected": -13.79401969909668,
+      "step": 250
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 2.7837259100642396e-08,
+      "logits/chosen": 2.4352571964263916,
+      "logits/rejected": 3.324097156524658,
+      "logps/chosen": -389.4182434082031,
+      "logps/rejected": -390.0982666015625,
+      "loss": 0.363,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.10746519267559052,
+      "rewards/margins": 14.993916511535645,
+      "rewards/rejected": -14.886451721191406,
+      "step": 260
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 2.890792291220557e-08,
+      "logits/chosen": 2.1370952129364014,
+      "logits/rejected": 3.407914400100708,
+      "logps/chosen": -434.8224182128906,
+      "logps/rejected": -440.6358947753906,
+      "loss": 0.4309,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 2.220722198486328,
+      "rewards/margins": 18.430742263793945,
+      "rewards/rejected": -16.210018157958984,
+      "step": 270
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 2.9978586723768736e-08,
+      "logits/chosen": 2.740478992462158,
+      "logits/rejected": 3.258228302001953,
+      "logps/chosen": -433.20782470703125,
+      "logps/rejected": -437.18719482421875,
+      "loss": 0.3487,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 2.553795576095581,
+      "rewards/margins": 19.554113388061523,
+      "rewards/rejected": -17.00031852722168,
+      "step": 280
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 3.1049250535331906e-08,
+      "logits/chosen": 2.1817595958709717,
+      "logits/rejected": 3.1689507961273193,
+      "logps/chosen": -440.03863525390625,
+      "logps/rejected": -409.65240478515625,
+      "loss": 0.3633,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 1.1603565216064453,
+      "rewards/margins": 17.322303771972656,
+      "rewards/rejected": -16.161945343017578,
+      "step": 290
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 3.211991434689507e-08,
+      "logits/chosen": 2.570457935333252,
+      "logits/rejected": 3.2502989768981934,
+      "logps/chosen": -454.56890869140625,
+      "logps/rejected": -416.1253356933594,
+      "loss": 0.4447,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 3.8409790992736816,
+      "rewards/margins": 22.155860900878906,
+      "rewards/rejected": -18.31488037109375,
+      "step": 300
+    },
+    {
+      "epoch": 0.19,
+      "eval_logits/chosen": 1.8170030117034912,
+      "eval_logits/rejected": 2.5056939125061035,
+      "eval_logps/chosen": -403.96087646484375,
+      "eval_logps/rejected": -405.89984130859375,
+      "eval_loss": 0.7104232311248779,
+      "eval_rewards/accuracies": 0.9296875,
+      "eval_rewards/chosen": 0.27033889293670654,
+      "eval_rewards/margins": 18.656814575195312,
+      "eval_rewards/rejected": -18.386476516723633,
+      "eval_runtime": 76.9022,
+      "eval_samples_per_second": 13.004,
+      "eval_steps_per_second": 0.416,
+      "step": 300
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 3.3190578158458246e-08,
+      "logits/chosen": 2.6424567699432373,
+      "logits/rejected": 3.5541133880615234,
+      "logps/chosen": -389.58636474609375,
+      "logps/rejected": -417.542236328125,
+      "loss": 0.1892,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 2.047499179840088,
+      "rewards/margins": 22.001834869384766,
+      "rewards/rejected": -19.954334259033203,
+      "step": 310
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 3.426124197002141e-08,
+      "logits/chosen": 2.256901264190674,
+      "logits/rejected": 3.4731545448303223,
+      "logps/chosen": -414.03875732421875,
+      "logps/rejected": -427.070556640625,
+      "loss": 0.3266,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.8677770495414734,
+      "rewards/margins": 22.491207122802734,
+      "rewards/rejected": -21.623430252075195,
+      "step": 320
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 3.533190578158458e-08,
+      "logits/chosen": 2.77763295173645,
+      "logits/rejected": 3.112457752227783,
+      "logps/chosen": -360.58135986328125,
+      "logps/rejected": -414.3995666503906,
+      "loss": 0.2844,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 2.416537046432495,
+      "rewards/margins": 22.439632415771484,
+      "rewards/rejected": -20.023096084594727,
+      "step": 330
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 3.640256959314775e-08,
+      "logits/chosen": 2.5908098220825195,
+      "logits/rejected": 3.282468795776367,
+      "logps/chosen": -449.01922607421875,
+      "logps/rejected": -422.73016357421875,
+      "loss": 0.3479,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 2.354691505432129,
+      "rewards/margins": 22.229778289794922,
+      "rewards/rejected": -19.875089645385742,
+      "step": 340
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 3.747323340471092e-08,
+      "logits/chosen": 2.534389019012451,
+      "logits/rejected": 3.5590693950653076,
+      "logps/chosen": -405.7064514160156,
+      "logps/rejected": -434.4012756347656,
+      "loss": 0.4823,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.6845805644989014,
+      "rewards/margins": 23.965158462524414,
+      "rewards/rejected": -22.280576705932617,
+      "step": 350
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 3.854389721627409e-08,
+      "logits/chosen": 2.5174474716186523,
+      "logits/rejected": 3.512732744216919,
+      "logps/chosen": -435.5006408691406,
+      "logps/rejected": -422.19366455078125,
+      "loss": 0.204,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 3.582836151123047,
+      "rewards/margins": 24.59599494934082,
+      "rewards/rejected": -21.013154983520508,
+      "step": 360
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 3.961456102783726e-08,
+      "logits/chosen": 2.3562188148498535,
+      "logits/rejected": 3.3465442657470703,
+      "logps/chosen": -470.4930725097656,
+      "logps/rejected": -405.6280822753906,
+      "loss": 0.3122,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 1.8608239889144897,
+      "rewards/margins": 23.556621551513672,
+      "rewards/rejected": -21.6957950592041,
+      "step": 370
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.068522483940043e-08,
+      "logits/chosen": 2.7507357597351074,
+      "logits/rejected": 3.5644543170928955,
+      "logps/chosen": -368.4208068847656,
+      "logps/rejected": -416.85302734375,
+      "loss": 0.2533,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 1.0723613500595093,
+      "rewards/margins": 24.53675651550293,
+      "rewards/rejected": -23.464397430419922,
+      "step": 380
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.175588865096359e-08,
+      "logits/chosen": 2.4924309253692627,
+      "logits/rejected": 3.5116946697235107,
+      "logps/chosen": -383.086181640625,
+      "logps/rejected": -417.8291931152344,
+      "loss": 0.2178,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 4.934910774230957,
+      "rewards/margins": 26.8948974609375,
+      "rewards/rejected": -21.959985733032227,
+      "step": 390
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.282655246252677e-08,
+      "logits/chosen": 2.5591559410095215,
+      "logits/rejected": 3.345097780227661,
+      "logps/chosen": -449.8941955566406,
+      "logps/rejected": -431.88818359375,
+      "loss": 0.2468,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 5.43572473526001,
+      "rewards/margins": 28.937246322631836,
+      "rewards/rejected": -23.501522064208984,
+      "step": 400
+    },
+    {
+      "epoch": 0.26,
+      "eval_logits/chosen": 1.8141372203826904,
+      "eval_logits/rejected": 2.503908395767212,
+      "eval_logps/chosen": -403.28955078125,
+      "eval_logps/rejected": -406.5526123046875,
+      "eval_loss": 0.5629487037658691,
+      "eval_rewards/accuracies": 0.9140625,
+      "eval_rewards/chosen": 3.626873731613159,
+      "eval_rewards/margins": 25.27718162536621,
+      "eval_rewards/rejected": -21.650310516357422,
+      "eval_runtime": 76.66,
+      "eval_samples_per_second": 13.045,
+      "eval_steps_per_second": 0.417,
+      "step": 400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.389721627408993e-08,
+      "logits/chosen": 2.353135585784912,
+      "logits/rejected": 3.1894092559814453,
+      "logps/chosen": -386.232666015625,
+      "logps/rejected": -399.92218017578125,
+      "loss": 0.1749,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 5.276652812957764,
+      "rewards/margins": 27.45583724975586,
+      "rewards/rejected": -22.17918586730957,
+      "step": 410
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.49678800856531e-08,
+      "logits/chosen": 1.99558424949646,
+      "logits/rejected": 3.1743838787078857,
+      "logps/chosen": -412.32110595703125,
+      "logps/rejected": -421.9241638183594,
+      "loss": 0.272,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 6.199848175048828,
+      "rewards/margins": 27.825729370117188,
+      "rewards/rejected": -21.62588119506836,
+      "step": 420
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.603854389721627e-08,
+      "logits/chosen": 2.605555772781372,
+      "logits/rejected": 3.407613754272461,
+      "logps/chosen": -434.2098083496094,
+      "logps/rejected": -447.4507751464844,
+      "loss": 0.3003,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 8.46845531463623,
+      "rewards/margins": 32.25395584106445,
+      "rewards/rejected": -23.785497665405273,
+      "step": 430
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.710920770877944e-08,
+      "logits/chosen": 2.1723690032958984,
+      "logits/rejected": 3.847750186920166,
+      "logps/chosen": -421.90155029296875,
+      "logps/rejected": -414.0150451660156,
+      "loss": 0.2367,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 8.460034370422363,
+      "rewards/margins": 31.530038833618164,
+      "rewards/rejected": -23.070003509521484,
+      "step": 440
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.817987152034261e-08,
+      "logits/chosen": 2.460693120956421,
+      "logits/rejected": 2.9349396228790283,
+      "logps/chosen": -502.804931640625,
+      "logps/rejected": -431.50335693359375,
+      "loss": 0.2369,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 7.705231666564941,
+      "rewards/margins": 32.00004959106445,
+      "rewards/rejected": -24.294815063476562,
+      "step": 450
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.925053533190578e-08,
+      "logits/chosen": 2.333646297454834,
+      "logits/rejected": 4.089064598083496,
+      "logps/chosen": -422.8240661621094,
+      "logps/rejected": -442.1726989746094,
+      "loss": 0.1691,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 4.384838581085205,
+      "rewards/margins": 29.339359283447266,
+      "rewards/rejected": -24.954524993896484,
+      "step": 460
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 5.032119914346895e-08,
+      "logits/chosen": 2.252077579498291,
+      "logits/rejected": 3.398280382156372,
+      "logps/chosen": -426.0081481933594,
+      "logps/rejected": -431.38623046875,
+      "loss": 0.2549,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 3.1347286701202393,
+      "rewards/margins": 29.128936767578125,
+      "rewards/rejected": -25.994205474853516,
+      "step": 470
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 5.139186295503212e-08,
+      "logits/chosen": 1.9574702978134155,
+      "logits/rejected": 3.0777688026428223,
+      "logps/chosen": -445.21435546875,
+      "logps/rejected": -431.9685974121094,
+      "loss": 0.2022,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 5.082431793212891,
+      "rewards/margins": 33.55067825317383,
+      "rewards/rejected": -28.468246459960938,
+      "step": 480
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 5.246252676659528e-08,
+      "logits/chosen": 2.4008655548095703,
+      "logits/rejected": 3.1396138668060303,
+      "logps/chosen": -396.2805480957031,
+      "logps/rejected": -431.75994873046875,
+      "loss": 0.1925,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 3.1321897506713867,
+      "rewards/margins": 31.742828369140625,
+      "rewards/rejected": -28.610637664794922,
+      "step": 490
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 5.353319057815846e-08,
+      "logits/chosen": 2.6065423488616943,
+      "logits/rejected": 3.01694917678833,
+      "logps/chosen": -441.64752197265625,
+      "logps/rejected": -450.52337646484375,
+      "loss": 0.238,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 5.9151458740234375,
+      "rewards/margins": 37.43492889404297,
+      "rewards/rejected": -31.5197811126709,
+      "step": 500
+    },
+    {
+      "epoch": 0.32,
+      "eval_logits/chosen": 1.8356637954711914,
+      "eval_logits/rejected": 2.518232822418213,
+      "eval_logps/chosen": -403.12066650390625,
+      "eval_logps/rejected": -407.75311279296875,
+      "eval_loss": 0.40245920419692993,
+      "eval_rewards/accuracies": 0.9296875,
+      "eval_rewards/chosen": 4.471410751342773,
+      "eval_rewards/margins": 32.124061584472656,
+      "eval_rewards/rejected": -27.652652740478516,
+      "eval_runtime": 76.7691,
+      "eval_samples_per_second": 13.026,
+      "eval_steps_per_second": 0.417,
+      "step": 500
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 5.460385438972163e-08,
+      "logits/chosen": 2.370246410369873,
+      "logits/rejected": 3.2074241638183594,
+      "logps/chosen": -463.4814453125,
+      "logps/rejected": -450.365478515625,
+      "loss": 0.1896,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 4.760863304138184,
+      "rewards/margins": 35.59904098510742,
+      "rewards/rejected": -30.838184356689453,
+      "step": 510
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 5.567451820128479e-08,
+      "logits/chosen": 2.4717459678649902,
+      "logits/rejected": 3.241302967071533,
+      "logps/chosen": -476.26690673828125,
+      "logps/rejected": -425.7784729003906,
+      "loss": 0.2277,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 5.998467922210693,
+      "rewards/margins": 37.100624084472656,
+      "rewards/rejected": -31.102157592773438,
+      "step": 520
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 5.6745182012847956e-08,
+      "logits/chosen": 2.8885834217071533,
+      "logits/rejected": 3.374998092651367,
+      "logps/chosen": -399.87249755859375,
+      "logps/rejected": -406.6166687011719,
+      "loss": 0.2215,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 5.892712116241455,
+      "rewards/margins": 36.44768524169922,
+      "rewards/rejected": -30.554973602294922,
+      "step": 530
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 5.781584582441114e-08,
+      "logits/chosen": 2.536708116531372,
+      "logits/rejected": 3.2403769493103027,
+      "logps/chosen": -429.4972229003906,
+      "logps/rejected": -423.7460021972656,
+      "loss": 0.1918,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 7.94210958480835,
+      "rewards/margins": 36.776371002197266,
+      "rewards/rejected": -28.83426284790039,
+      "step": 540
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 5.88865096359743e-08,
+      "logits/chosen": 1.9649451971054077,
+      "logits/rejected": 3.1596691608428955,
+      "logps/chosen": -472.22119140625,
+      "logps/rejected": -425.32763671875,
+      "loss": 0.15,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 12.239465713500977,
+      "rewards/margins": 48.21662139892578,
+      "rewards/rejected": -35.977149963378906,
+      "step": 550
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 5.995717344753747e-08,
+      "logits/chosen": 3.032383441925049,
+      "logits/rejected": 3.257586717605591,
+      "logps/chosen": -416.17071533203125,
+      "logps/rejected": -410.1018981933594,
+      "loss": 0.1132,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 7.306866645812988,
+      "rewards/margins": 37.124568939208984,
+      "rewards/rejected": -29.81770896911621,
+      "step": 560
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.102783725910064e-08,
+      "logits/chosen": 2.7644500732421875,
+      "logits/rejected": 3.5682806968688965,
+      "logps/chosen": -383.52093505859375,
+      "logps/rejected": -394.8160705566406,
+      "loss": 0.2133,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 5.922687530517578,
+      "rewards/margins": 37.88574981689453,
+      "rewards/rejected": -31.963062286376953,
+      "step": 570
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.209850107066381e-08,
+      "logits/chosen": 2.341892957687378,
+      "logits/rejected": 3.260730028152466,
+      "logps/chosen": -394.4947814941406,
+      "logps/rejected": -405.29498291015625,
+      "loss": 0.082,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 6.424149990081787,
+      "rewards/margins": 38.39518737792969,
+      "rewards/rejected": -31.971033096313477,
+      "step": 580
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.316916488222698e-08,
+      "logits/chosen": 2.526381015777588,
+      "logits/rejected": 3.192876100540161,
+      "logps/chosen": -426.3685607910156,
+      "logps/rejected": -403.46759033203125,
+      "loss": 0.1342,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 3.9063773155212402,
+      "rewards/margins": 39.68169021606445,
+      "rewards/rejected": -35.77531814575195,
+      "step": 590
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.423982869379014e-08,
+      "logits/chosen": 2.456986427307129,
+      "logits/rejected": 2.974177837371826,
+      "logps/chosen": -430.700927734375,
+      "logps/rejected": -460.1724548339844,
+      "loss": 0.0835,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 2.9916865825653076,
+      "rewards/margins": 47.707908630371094,
+      "rewards/rejected": -44.716224670410156,
+      "step": 600
+    },
+    {
+      "epoch": 0.39,
+      "eval_logits/chosen": 1.8376859426498413,
+      "eval_logits/rejected": 2.519270658493042,
+      "eval_logps/chosen": -404.12213134765625,
+      "eval_logps/rejected": -409.9280700683594,
+      "eval_loss": 0.5330207943916321,
+      "eval_rewards/accuracies": 0.9453125,
+      "eval_rewards/chosen": -0.5357855558395386,
+      "eval_rewards/margins": 37.99186325073242,
+      "eval_rewards/rejected": -38.52764892578125,
+      "eval_runtime": 76.8308,
+      "eval_samples_per_second": 13.016,
+      "eval_steps_per_second": 0.416,
+      "step": 600
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.531049250535332e-08,
+      "logits/chosen": 2.4591498374938965,
+      "logits/rejected": 3.2855117321014404,
+      "logps/chosen": -401.3484802246094,
+      "logps/rejected": -434.5301208496094,
+      "loss": 0.142,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.02256155014038086,
+      "rewards/margins": 46.42057418823242,
+      "rewards/rejected": -46.443138122558594,
+      "step": 610
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.638115631691649e-08,
+      "logits/chosen": 2.539071559906006,
+      "logits/rejected": 3.2421278953552246,
+      "logps/chosen": -442.18243408203125,
+      "logps/rejected": -444.18878173828125,
+      "loss": 0.1058,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.043628692626953,
+      "rewards/margins": 43.48805618286133,
+      "rewards/rejected": -41.444427490234375,
+      "step": 620
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.745182012847965e-08,
+      "logits/chosen": 2.5688178539276123,
+      "logits/rejected": 3.445737838745117,
+      "logps/chosen": -411.5978088378906,
+      "logps/rejected": -407.0299072265625,
+      "loss": 0.1525,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 1.022148609161377,
+      "rewards/margins": 39.893680572509766,
+      "rewards/rejected": -38.87153244018555,
+      "step": 630
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.852248394004282e-08,
+      "logits/chosen": 2.5901875495910645,
+      "logits/rejected": 2.911647319793701,
+      "logps/chosen": -420.1778869628906,
+      "logps/rejected": -382.272216796875,
+      "loss": 0.2718,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 3.5251822471618652,
+      "rewards/margins": 41.309791564941406,
+      "rewards/rejected": -37.784610748291016,
+      "step": 640
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 6.9593147751606e-08,
+      "logits/chosen": 2.287266492843628,
+      "logits/rejected": 3.33722186088562,
+      "logps/chosen": -452.6976013183594,
+      "logps/rejected": -439.2931213378906,
+      "loss": 0.0169,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 6.000566482543945,
+      "rewards/margins": 46.02397918701172,
+      "rewards/rejected": -40.023414611816406,
+      "step": 650
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 7.066381156316916e-08,
+      "logits/chosen": 2.2046732902526855,
+      "logits/rejected": 3.6667628288269043,
+      "logps/chosen": -403.92205810546875,
+      "logps/rejected": -408.7096862792969,
+      "loss": 0.2138,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 6.08463191986084,
+      "rewards/margins": 44.13505554199219,
+      "rewards/rejected": -38.0504264831543,
+      "step": 660
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 7.173447537473233e-08,
+      "logits/chosen": 2.4322450160980225,
+      "logits/rejected": 3.5664360523223877,
+      "logps/chosen": -416.0870056152344,
+      "logps/rejected": -415.59197998046875,
+      "loss": 0.1237,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 4.357824325561523,
+      "rewards/margins": 44.89923858642578,
+      "rewards/rejected": -40.541419982910156,
+      "step": 670
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 7.28051391862955e-08,
+      "logits/chosen": 2.468834638595581,
+      "logits/rejected": 3.187020778656006,
+      "logps/chosen": -395.1986083984375,
+      "logps/rejected": -432.7027282714844,
+      "loss": 0.1312,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 7.26413631439209,
+      "rewards/margins": 49.74661636352539,
+      "rewards/rejected": -42.48247528076172,
+      "step": 680
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 7.387580299785867e-08,
+      "logits/chosen": 2.341510772705078,
+      "logits/rejected": 3.5096206665039062,
+      "logps/chosen": -391.4788513183594,
+      "logps/rejected": -392.6119384765625,
+      "loss": 0.1998,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 6.877418518066406,
+      "rewards/margins": 45.81779479980469,
+      "rewards/rejected": -38.94037628173828,
+      "step": 690
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 7.494646680942184e-08,
+      "logits/chosen": 2.7313547134399414,
+      "logits/rejected": 3.659938097000122,
+      "logps/chosen": -375.4259033203125,
+      "logps/rejected": -431.1378479003906,
+      "loss": 0.1294,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 4.9859724044799805,
+      "rewards/margins": 47.63801574707031,
+      "rewards/rejected": -42.65203857421875,
+      "step": 700
+    },
+    {
+      "epoch": 0.45,
+      "eval_logits/chosen": 1.8345167636871338,
+      "eval_logits/rejected": 2.522487163543701,
+      "eval_logps/chosen": -403.2203674316406,
+      "eval_logps/rejected": -410.374755859375,
+      "eval_loss": 0.40866366028785706,
+      "eval_rewards/accuracies": 0.953125,
+      "eval_rewards/chosen": 3.9727911949157715,
+      "eval_rewards/margins": 44.73374938964844,
+      "eval_rewards/rejected": -40.760955810546875,
+      "eval_runtime": 76.9261,
+      "eval_samples_per_second": 12.999,
+      "eval_steps_per_second": 0.416,
+      "step": 700
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 7.601713062098501e-08,
+      "logits/chosen": 2.3793327808380127,
+      "logits/rejected": 3.142347574234009,
+      "logps/chosen": -423.20880126953125,
+      "logps/rejected": -401.803955078125,
+      "loss": 0.1473,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 8.15241527557373,
+      "rewards/margins": 47.408851623535156,
+      "rewards/rejected": -39.256431579589844,
+      "step": 710
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 7.708779443254818e-08,
+      "logits/chosen": 2.7316575050354004,
+      "logits/rejected": 3.029636859893799,
+      "logps/chosen": -434.23480224609375,
+      "logps/rejected": -403.28717041015625,
+      "loss": 0.1684,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 10.977544784545898,
+      "rewards/margins": 50.05615234375,
+      "rewards/rejected": -39.07860565185547,
+      "step": 720
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 7.815845824411135e-08,
+      "logits/chosen": 2.5688631534576416,
+      "logits/rejected": 3.0123863220214844,
+      "logps/chosen": -449.0638732910156,
+      "logps/rejected": -398.37152099609375,
+      "loss": 0.0823,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 8.119828224182129,
+      "rewards/margins": 45.32478332519531,
+      "rewards/rejected": -37.2049560546875,
+      "step": 730
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 7.922912205567452e-08,
+      "logits/chosen": 2.6267566680908203,
+      "logits/rejected": 3.3151562213897705,
+      "logps/chosen": -390.72698974609375,
+      "logps/rejected": -381.4679260253906,
+      "loss": 0.2223,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 6.810037136077881,
+      "rewards/margins": 40.27370071411133,
+      "rewards/rejected": -33.46365737915039,
+      "step": 740
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 8.029978586723767e-08,
+      "logits/chosen": 2.355883836746216,
+      "logits/rejected": 3.5080723762512207,
+      "logps/chosen": -395.70184326171875,
+      "logps/rejected": -403.0619812011719,
+      "loss": 0.1436,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 9.782814025878906,
+      "rewards/margins": 49.49211883544922,
+      "rewards/rejected": -39.70930480957031,
+      "step": 750
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 8.137044967880086e-08,
+      "logits/chosen": 2.538801670074463,
+      "logits/rejected": 2.962019205093384,
+      "logps/chosen": -395.83062744140625,
+      "logps/rejected": -439.6985778808594,
+      "loss": 0.1789,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 6.836484432220459,
+      "rewards/margins": 59.103851318359375,
+      "rewards/rejected": -52.267372131347656,
+      "step": 760
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 8.244111349036403e-08,
+      "logits/chosen": 2.372373104095459,
+      "logits/rejected": 3.526844024658203,
+      "logps/chosen": -391.49090576171875,
+      "logps/rejected": -436.17193603515625,
+      "loss": 0.1816,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 1.2613162994384766,
+      "rewards/margins": 54.581298828125,
+      "rewards/rejected": -53.31998825073242,
+      "step": 770
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 8.351177730192718e-08,
+      "logits/chosen": 2.641442060470581,
+      "logits/rejected": 3.504525661468506,
+      "logps/chosen": -429.4436950683594,
+      "logps/rejected": -426.1357421875,
+      "loss": 0.0799,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.4186324179172516,
+      "rewards/margins": 56.703216552734375,
+      "rewards/rejected": -57.121849060058594,
+      "step": 780
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 8.458244111349035e-08,
+      "logits/chosen": 2.8256888389587402,
+      "logits/rejected": 3.2680656909942627,
+      "logps/chosen": -432.7039489746094,
+      "logps/rejected": -435.9918518066406,
+      "loss": 0.1115,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19469785690307617,
+      "rewards/margins": 58.439208984375,
+      "rewards/rejected": -58.2445182800293,
+      "step": 790
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 8.565310492505354e-08,
+      "logits/chosen": 2.2154085636138916,
+      "logits/rejected": 3.348832607269287,
+      "logps/chosen": -443.20269775390625,
+      "logps/rejected": -411.3150939941406,
+      "loss": 0.2809,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 7.068073272705078,
+      "rewards/margins": 56.71687698364258,
+      "rewards/rejected": -49.648807525634766,
+      "step": 800
+    },
+    {
+      "epoch": 0.51,
+      "eval_logits/chosen": 1.8377827405929565,
+      "eval_logits/rejected": 2.5198471546173096,
+      "eval_logps/chosen": -403.1432189941406,
+      "eval_logps/rejected": -412.51531982421875,
+      "eval_loss": 0.5127639770507812,
+      "eval_rewards/accuracies": 0.953125,
+      "eval_rewards/chosen": 4.35873556137085,
+      "eval_rewards/margins": 55.822689056396484,
+      "eval_rewards/rejected": -51.463958740234375,
+      "eval_runtime": 76.8497,
+      "eval_samples_per_second": 13.012,
+      "eval_steps_per_second": 0.416,
+      "step": 800
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 8.672376873661669e-08,
+      "logits/chosen": 2.567979335784912,
+      "logits/rejected": 3.7970213890075684,
+      "logps/chosen": -379.3563232421875,
+      "logps/rejected": -417.9363708496094,
+      "loss": 0.1456,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 6.566949367523193,
+      "rewards/margins": 58.88117218017578,
+      "rewards/rejected": -52.31422805786133,
+      "step": 810
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 8.779443254817986e-08,
+      "logits/chosen": 2.987666130065918,
+      "logits/rejected": 3.0821828842163086,
+      "logps/chosen": -375.0899353027344,
+      "logps/rejected": -380.730712890625,
+      "loss": 0.113,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 4.425510406494141,
+      "rewards/margins": 57.55885696411133,
+      "rewards/rejected": -53.13335037231445,
+      "step": 820
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 8.886509635974304e-08,
+      "logits/chosen": 2.3225650787353516,
+      "logits/rejected": 3.374905824661255,
+      "logps/chosen": -404.91314697265625,
+      "logps/rejected": -406.26690673828125,
+      "loss": 0.221,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.956459641456604,
+      "rewards/margins": 58.1107177734375,
+      "rewards/rejected": -57.15425491333008,
+      "step": 830
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 8.99357601713062e-08,
+      "logits/chosen": 2.3407421112060547,
+      "logits/rejected": 3.029287815093994,
+      "logps/chosen": -389.4285583496094,
+      "logps/rejected": -422.2571716308594,
+      "loss": 0.123,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 5.533617973327637,
+      "rewards/margins": 57.36924362182617,
+      "rewards/rejected": -51.83562469482422,
+      "step": 840
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 9.100642398286937e-08,
+      "logits/chosen": 2.4234728813171387,
+      "logits/rejected": 3.6180338859558105,
+      "logps/chosen": -384.3294982910156,
+      "logps/rejected": -382.1348876953125,
+      "loss": 0.1292,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 5.42165994644165,
+      "rewards/margins": 53.802391052246094,
+      "rewards/rejected": -48.38072967529297,
+      "step": 850
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 9.207708779443254e-08,
+      "logits/chosen": 2.336247444152832,
+      "logits/rejected": 3.8205437660217285,
+      "logps/chosen": -427.49468994140625,
+      "logps/rejected": -434.2718811035156,
+      "loss": 0.0955,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 17.532028198242188,
+      "rewards/margins": 70.969482421875,
+      "rewards/rejected": -53.437461853027344,
+      "step": 860
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 9.314775160599571e-08,
+      "logits/chosen": 2.201402187347412,
+      "logits/rejected": 3.6681246757507324,
+      "logps/chosen": -437.94091796875,
+      "logps/rejected": -429.28021240234375,
+      "loss": 0.0989,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 10.197607040405273,
+      "rewards/margins": 63.349510192871094,
+      "rewards/rejected": -53.15190505981445,
+      "step": 870
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 9.421841541755888e-08,
+      "logits/chosen": 2.64699125289917,
+      "logits/rejected": 2.797672986984253,
+      "logps/chosen": -419.5973205566406,
+      "logps/rejected": -427.1859436035156,
+      "loss": 0.2619,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 5.267810344696045,
+      "rewards/margins": 64.8418197631836,
+      "rewards/rejected": -59.573997497558594,
+      "step": 880
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 9.528907922912205e-08,
+      "logits/chosen": 2.1525919437408447,
+      "logits/rejected": 3.06669545173645,
+      "logps/chosen": -447.0118103027344,
+      "logps/rejected": -425.5939025878906,
+      "loss": 0.1568,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 12.983186721801758,
+      "rewards/margins": 62.67449188232422,
+      "rewards/rejected": -49.69130325317383,
+      "step": 890
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 9.635974304068522e-08,
+      "logits/chosen": 2.4299418926239014,
+      "logits/rejected": 3.4242959022521973,
+      "logps/chosen": -390.6571350097656,
+      "logps/rejected": -455.71331787109375,
+      "loss": 0.2051,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 5.089010715484619,
+      "rewards/margins": 74.85060119628906,
+      "rewards/rejected": -69.76158905029297,
+      "step": 900
+    },
+    {
+      "epoch": 0.58,
+      "eval_logits/chosen": 1.8021771907806396,
+      "eval_logits/rejected": 2.491039752960205,
+      "eval_logps/chosen": -400.8381042480469,
+      "eval_logps/rejected": -412.0938720703125,
+      "eval_loss": 0.5601515173912048,
+      "eval_rewards/accuracies": 0.9375,
+      "eval_rewards/chosen": 15.884079933166504,
+      "eval_rewards/margins": 65.24065399169922,
+      "eval_rewards/rejected": -49.3565673828125,
+      "eval_runtime": 77.0047,
+      "eval_samples_per_second": 12.986,
+      "eval_steps_per_second": 0.416,
+      "step": 900
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 9.743040685224839e-08,
+      "logits/chosen": 2.6785802841186523,
+      "logits/rejected": 3.158878803253174,
+      "logps/chosen": -364.9392395019531,
+      "logps/rejected": -421.75604248046875,
+      "loss": 0.1121,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 15.941950798034668,
+      "rewards/margins": 70.44316101074219,
+      "rewards/rejected": -54.501220703125,
+      "step": 910
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 9.850107066381156e-08,
+      "logits/chosen": 2.1345481872558594,
+      "logits/rejected": 3.1029703617095947,
+      "logps/chosen": -467.07318115234375,
+      "logps/rejected": -428.45220947265625,
+      "loss": 0.2077,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 8.983004570007324,
+      "rewards/margins": 70.48181915283203,
+      "rewards/rejected": -61.49879837036133,
+      "step": 920
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 9.957173447537473e-08,
+      "logits/chosen": 2.542916774749756,
+      "logits/rejected": 3.5035908222198486,
+      "logps/chosen": -423.0205993652344,
+      "logps/rejected": -416.65509033203125,
+      "loss": 0.2814,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 5.553844928741455,
+      "rewards/margins": 68.67607116699219,
+      "rewards/rejected": -63.122222900390625,
+      "step": 930
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 9.992858843132586e-08,
+      "logits/chosen": 2.491159439086914,
+      "logits/rejected": 3.259873867034912,
+      "logps/chosen": -475.2666931152344,
+      "logps/rejected": -461.07061767578125,
+      "loss": 0.1522,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 4.259446144104004,
+      "rewards/margins": 78.9701919555664,
+      "rewards/rejected": -74.71073913574219,
+      "step": 940
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 9.980956915020233e-08,
+      "logits/chosen": 2.68623423576355,
+      "logits/rejected": 3.0718812942504883,
+      "logps/chosen": -361.89483642578125,
+      "logps/rejected": -413.39288330078125,
+      "loss": 0.3246,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 8.68491268157959,
+      "rewards/margins": 75.4022445678711,
+      "rewards/rejected": -66.71733093261719,
+      "step": 950
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 9.969054986907879e-08,
+      "logits/chosen": 2.752331495285034,
+      "logits/rejected": 3.481398344039917,
+      "logps/chosen": -392.8135986328125,
+      "logps/rejected": -413.65313720703125,
+      "loss": 0.2217,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.2605032920837402,
+      "rewards/margins": 72.24468994140625,
+      "rewards/rejected": -68.98418426513672,
+      "step": 960
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 9.957153058795524e-08,
+      "logits/chosen": 2.0877063274383545,
+      "logits/rejected": 3.1967997550964355,
+      "logps/chosen": -398.50030517578125,
+      "logps/rejected": -458.2386779785156,
+      "loss": 0.1414,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 4.164697170257568,
+      "rewards/margins": 79.06629943847656,
+      "rewards/rejected": -74.90159606933594,
+      "step": 970
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 9.94525113068317e-08,
+      "logits/chosen": 2.6983773708343506,
+      "logits/rejected": 3.458531141281128,
+      "logps/chosen": -400.029296875,
+      "logps/rejected": -411.06903076171875,
+      "loss": 0.2352,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 4.437505722045898,
+      "rewards/margins": 78.08453369140625,
+      "rewards/rejected": -73.64701843261719,
+      "step": 980
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 9.933349202570817e-08,
+      "logits/chosen": 2.2401540279388428,
+      "logits/rejected": 3.9224307537078857,
+      "logps/chosen": -416.910400390625,
+      "logps/rejected": -441.75482177734375,
+      "loss": 0.1331,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 7.2108893394470215,
+      "rewards/margins": 89.5989990234375,
+      "rewards/rejected": -82.38809967041016,
+      "step": 990
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 9.921447274458463e-08,
+      "logits/chosen": 2.333087682723999,
+      "logits/rejected": 3.3145065307617188,
+      "logps/chosen": -430.5704650878906,
+      "logps/rejected": -428.00384521484375,
+      "loss": 0.2009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.077216863632202,
+      "rewards/margins": 81.33201599121094,
+      "rewards/rejected": -84.40922546386719,
+      "step": 1000
+    },
+    {
+      "epoch": 0.64,
+      "eval_logits/chosen": 1.85965096950531,
+      "eval_logits/rejected": 2.5712876319885254,
+      "eval_logps/chosen": -405.8722839355469,
+      "eval_logps/rejected": -420.0833740234375,
+      "eval_loss": 0.873622715473175,
+      "eval_rewards/accuracies": 0.9296875,
+      "eval_rewards/chosen": -9.286641120910645,
+      "eval_rewards/margins": 80.01751708984375,
+      "eval_rewards/rejected": -89.30416107177734,
+      "eval_runtime": 76.9649,
+      "eval_samples_per_second": 12.993,
+      "eval_steps_per_second": 0.416,
+      "step": 1000
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 9.909545346346108e-08,
+      "logits/chosen": 2.4723129272460938,
+      "logits/rejected": 3.478045701980591,
+      "logps/chosen": -388.63848876953125,
+      "logps/rejected": -405.98748779296875,
+      "loss": 0.3397,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 3.27624773979187,
+      "rewards/margins": 83.57379150390625,
+      "rewards/rejected": -80.29755401611328,
+      "step": 1010
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 9.897643418233753e-08,
+      "logits/chosen": 2.1651980876922607,
+      "logits/rejected": 3.202363967895508,
+      "logps/chosen": -439.97894287109375,
+      "logps/rejected": -464.5369567871094,
+      "loss": 0.296,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 9.399145126342773,
+      "rewards/margins": 104.0501480102539,
+      "rewards/rejected": -94.65100860595703,
+      "step": 1020
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 9.885741490121398e-08,
+      "logits/chosen": 2.6700689792633057,
+      "logits/rejected": 3.753943681716919,
+      "logps/chosen": -412.857666015625,
+      "logps/rejected": -408.2137756347656,
+      "loss": 0.267,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 17.997615814208984,
+      "rewards/margins": 86.39213562011719,
+      "rewards/rejected": -68.39451599121094,
+      "step": 1030
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 9.873839562009045e-08,
+      "logits/chosen": 2.110105514526367,
+      "logits/rejected": 3.5830421447753906,
+      "logps/chosen": -426.0606994628906,
+      "logps/rejected": -430.07073974609375,
+      "loss": 0.1166,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 16.306238174438477,
+      "rewards/margins": 97.02738952636719,
+      "rewards/rejected": -80.72114562988281,
+      "step": 1040
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 9.861937633896691e-08,
+      "logits/chosen": 2.830747127532959,
+      "logits/rejected": 3.9411635398864746,
+      "logps/chosen": -391.10858154296875,
+      "logps/rejected": -417.4032287597656,
+      "loss": 0.3818,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 4.93004846572876,
+      "rewards/margins": 88.80459594726562,
+      "rewards/rejected": -83.87454223632812,
+      "step": 1050
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 9.850035705784336e-08,
+      "logits/chosen": 2.3416190147399902,
+      "logits/rejected": 3.0985264778137207,
+      "logps/chosen": -461.04718017578125,
+      "logps/rejected": -448.1722106933594,
+      "loss": 0.0969,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 14.76380443572998,
+      "rewards/margins": 97.76150512695312,
+      "rewards/rejected": -82.99769592285156,
+      "step": 1060
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 9.838133777671982e-08,
+      "logits/chosen": 2.414294719696045,
+      "logits/rejected": 3.4742074012756348,
+      "logps/chosen": -384.5827941894531,
+      "logps/rejected": -398.83050537109375,
+      "loss": 0.4072,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 10.109460830688477,
+      "rewards/margins": 92.21754455566406,
+      "rewards/rejected": -82.10807037353516,
+      "step": 1070
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 9.826231849559629e-08,
+      "logits/chosen": 2.7465944290161133,
+      "logits/rejected": 3.1911470890045166,
+      "logps/chosen": -422.5624084472656,
+      "logps/rejected": -440.286376953125,
+      "loss": 0.1751,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 17.269311904907227,
+      "rewards/margins": 104.30892181396484,
+      "rewards/rejected": -87.03959655761719,
+      "step": 1080
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 9.814329921447275e-08,
+      "logits/chosen": 2.822819232940674,
+      "logits/rejected": 3.005201816558838,
+      "logps/chosen": -391.0602111816406,
+      "logps/rejected": -419.813720703125,
+      "loss": 0.3021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 13.785235404968262,
+      "rewards/margins": 121.23738098144531,
+      "rewards/rejected": -107.45216369628906,
+      "step": 1090
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 9.80242799333492e-08,
+      "logits/chosen": 2.8149497509002686,
+      "logits/rejected": 3.3617329597473145,
+      "logps/chosen": -402.799560546875,
+      "logps/rejected": -425.45343017578125,
+      "loss": 0.2838,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 14.5247163772583,
+      "rewards/margins": 105.98973083496094,
+      "rewards/rejected": -91.46500396728516,
+      "step": 1100
+    },
+    {
+      "epoch": 0.71,
+      "eval_logits/chosen": 1.8558049201965332,
+      "eval_logits/rejected": 2.5366647243499756,
+      "eval_logps/chosen": -402.8280944824219,
+      "eval_logps/rejected": -421.0693054199219,
+      "eval_loss": 0.8600718975067139,
+      "eval_rewards/accuracies": 0.953125,
+      "eval_rewards/chosen": 5.934433460235596,
+      "eval_rewards/margins": 100.16818237304688,
+      "eval_rewards/rejected": -94.23374938964844,
+      "eval_runtime": 76.8109,
+      "eval_samples_per_second": 13.019,
+      "eval_steps_per_second": 0.417,
+      "step": 1100
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 9.790526065222565e-08,
+      "logits/chosen": 2.653266429901123,
+      "logits/rejected": 3.1961002349853516,
+      "logps/chosen": -445.80322265625,
+      "logps/rejected": -429.64373779296875,
+      "loss": 0.5293,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 21.07932472229004,
+      "rewards/margins": 100.654052734375,
+      "rewards/rejected": -79.57472229003906,
+      "step": 1110
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 9.778624137110211e-08,
+      "logits/chosen": 2.4288971424102783,
+      "logits/rejected": 3.4981236457824707,
+      "logps/chosen": -452.72967529296875,
+      "logps/rejected": -437.428955078125,
+      "loss": 0.1353,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 19.18131446838379,
+      "rewards/margins": 117.1941146850586,
+      "rewards/rejected": -98.01280212402344,
+      "step": 1120
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 9.766722208997857e-08,
+      "logits/chosen": 2.525935173034668,
+      "logits/rejected": 3.092148780822754,
+      "logps/chosen": -402.27886962890625,
+      "logps/rejected": -428.43280029296875,
+      "loss": 0.1894,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 20.851654052734375,
+      "rewards/margins": 118.8115234375,
+      "rewards/rejected": -97.95986938476562,
+      "step": 1130
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 9.754820280885503e-08,
+      "logits/chosen": 2.715815782546997,
+      "logits/rejected": 4.007362365722656,
+      "logps/chosen": -399.8186340332031,
+      "logps/rejected": -423.14862060546875,
+      "loss": 0.1684,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 9.72397232055664,
+      "rewards/margins": 105.40447998046875,
+      "rewards/rejected": -95.68050384521484,
+      "step": 1140
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.742918352773148e-08,
+      "logits/chosen": 2.2262580394744873,
+      "logits/rejected": 3.2408432960510254,
+      "logps/chosen": -477.47711181640625,
+      "logps/rejected": -495.76043701171875,
+      "loss": 0.1725,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 6.365964412689209,
+      "rewards/margins": 132.43780517578125,
+      "rewards/rejected": -126.07185363769531,
+      "step": 1150
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.731016424660795e-08,
+      "logits/chosen": 2.7380359172821045,
+      "logits/rejected": 3.5110669136047363,
+      "logps/chosen": -398.9952392578125,
+      "logps/rejected": -426.838623046875,
+      "loss": 0.2338,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 12.770757675170898,
+      "rewards/margins": 123.94891357421875,
+      "rewards/rejected": -111.17814636230469,
+      "step": 1160
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.719114496548441e-08,
+      "logits/chosen": 2.0883517265319824,
+      "logits/rejected": 3.5194995403289795,
+      "logps/chosen": -424.92803955078125,
+      "logps/rejected": -390.85784912109375,
+      "loss": 0.3672,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 9.357166290283203,
+      "rewards/margins": 103.42280578613281,
+      "rewards/rejected": -94.06563568115234,
+      "step": 1170
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 9.707212568436087e-08,
+      "logits/chosen": 2.170081615447998,
+      "logits/rejected": 3.3753578662872314,
+      "logps/chosen": -437.91082763671875,
+      "logps/rejected": -430.90966796875,
+      "loss": 0.0796,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 8.644025802612305,
+      "rewards/margins": 108.9352798461914,
+      "rewards/rejected": -100.2912368774414,
+      "step": 1180
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 9.695310640323732e-08,
+      "logits/chosen": 2.7604737281799316,
+      "logits/rejected": 3.7521843910217285,
+      "logps/chosen": -370.9176025390625,
+      "logps/rejected": -435.0440979003906,
+      "loss": 0.2187,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 10.98758316040039,
+      "rewards/margins": 134.98072814941406,
+      "rewards/rejected": -123.9931411743164,
+      "step": 1190
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 9.683408712211378e-08,
+      "logits/chosen": 2.3693931102752686,
+      "logits/rejected": 3.17862606048584,
+      "logps/chosen": -467.9139099121094,
+      "logps/rejected": -460.11297607421875,
+      "loss": 0.3489,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 3.4328982830047607,
+      "rewards/margins": 135.82644653320312,
+      "rewards/rejected": -132.3935546875,
+      "step": 1200
+    },
+    {
+      "epoch": 0.77,
+      "eval_logits/chosen": 1.8742191791534424,
+      "eval_logits/rejected": 2.560530662536621,
+      "eval_logps/chosen": -404.9606628417969,
+      "eval_logps/rejected": -424.3955383300781,
+      "eval_loss": 1.1546683311462402,
+      "eval_rewards/accuracies": 0.9453125,
+      "eval_rewards/chosen": -4.728623390197754,
+      "eval_rewards/margins": 106.13644409179688,
+      "eval_rewards/rejected": -110.86505889892578,
+      "eval_runtime": 77.052,
+      "eval_samples_per_second": 12.978,
+      "eval_steps_per_second": 0.415,
+      "step": 1200
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 9.671506784099024e-08,
+      "logits/chosen": 2.997239589691162,
+      "logits/rejected": 3.5606212615966797,
+      "logps/chosen": -392.54229736328125,
+      "logps/rejected": -423.11322021484375,
+      "loss": 0.0937,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.05036654323339462,
+      "rewards/margins": 117.02787780761719,
+      "rewards/rejected": -117.07823181152344,
+      "step": 1210
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 9.659604855986669e-08,
+      "logits/chosen": 2.1404788494110107,
+      "logits/rejected": 3.323446273803711,
+      "logps/chosen": -427.2289123535156,
+      "logps/rejected": -433.6624450683594,
+      "loss": 0.2752,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 12.247556686401367,
+      "rewards/margins": 107.3965835571289,
+      "rewards/rejected": -95.14904022216797,
+      "step": 1220
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 9.647702927874315e-08,
+      "logits/chosen": 2.563352346420288,
+      "logits/rejected": 3.0540013313293457,
+      "logps/chosen": -447.8446350097656,
+      "logps/rejected": -452.87969970703125,
+      "loss": 0.0435,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 23.387487411499023,
+      "rewards/margins": 119.50389099121094,
+      "rewards/rejected": -96.11639404296875,
+      "step": 1230
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.63580099976196e-08,
+      "logits/chosen": 2.1849985122680664,
+      "logits/rejected": 3.4553771018981934,
+      "logps/chosen": -387.8475341796875,
+      "logps/rejected": -393.53192138671875,
+      "loss": 0.2143,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 26.278972625732422,
+      "rewards/margins": 115.37886810302734,
+      "rewards/rejected": -89.09989929199219,
+      "step": 1240
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.623899071649607e-08,
+      "logits/chosen": 3.139633893966675,
+      "logits/rejected": 3.37255859375,
+      "logps/chosen": -398.74005126953125,
+      "logps/rejected": -418.0592346191406,
+      "loss": 0.0799,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 22.002050399780273,
+      "rewards/margins": 125.72208404541016,
+      "rewards/rejected": -103.72003173828125,
+      "step": 1250
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.611997143537253e-08,
+      "logits/chosen": 2.665964126586914,
+      "logits/rejected": 2.8448374271392822,
+      "logps/chosen": -427.25677490234375,
+      "logps/rejected": -430.95025634765625,
+      "loss": 0.3788,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 21.40536880493164,
+      "rewards/margins": 111.3967056274414,
+      "rewards/rejected": -89.9913330078125,
+      "step": 1260
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.600095215424899e-08,
+      "logits/chosen": 2.7928450107574463,
+      "logits/rejected": 3.595740795135498,
+      "logps/chosen": -382.04327392578125,
+      "logps/rejected": -431.33734130859375,
+      "loss": 0.3577,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 16.352216720581055,
+      "rewards/margins": 119.71681213378906,
+      "rewards/rejected": -103.3646011352539,
+      "step": 1270
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.588193287312544e-08,
+      "logits/chosen": 2.237015962600708,
+      "logits/rejected": 2.9541878700256348,
+      "logps/chosen": -421.40032958984375,
+      "logps/rejected": -427.09649658203125,
+      "loss": 0.0439,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 19.52680015563965,
+      "rewards/margins": 119.0426254272461,
+      "rewards/rejected": -99.51582336425781,
+      "step": 1280
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.57629135920019e-08,
+      "logits/chosen": 2.776695728302002,
+      "logits/rejected": 3.7990450859069824,
+      "logps/chosen": -398.505859375,
+      "logps/rejected": -432.1246032714844,
+      "loss": 0.5895,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 19.98464012145996,
+      "rewards/margins": 124.7027816772461,
+      "rewards/rejected": -104.7181625366211,
+      "step": 1290
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 9.564389431087836e-08,
+      "logits/chosen": 2.5397887229919434,
+      "logits/rejected": 3.605865478515625,
+      "logps/chosen": -403.42132568359375,
+      "logps/rejected": -455.809814453125,
+      "loss": 0.1571,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 29.50467872619629,
+      "rewards/margins": 132.95152282714844,
+      "rewards/rejected": -103.44685363769531,
+      "step": 1300
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": 1.7718256711959839,
+      "eval_logits/rejected": 2.484117031097412,
+      "eval_logps/chosen": -400.06805419921875,
+      "eval_logps/rejected": -421.99560546875,
+      "eval_loss": 0.9890532493591309,
+      "eval_rewards/accuracies": 0.9453125,
+      "eval_rewards/chosen": 19.734703063964844,
+      "eval_rewards/margins": 118.60000610351562,
+      "eval_rewards/rejected": -98.86530303955078,
+      "eval_runtime": 76.9799,
+      "eval_samples_per_second": 12.99,
+      "eval_steps_per_second": 0.416,
+      "step": 1300
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 9.552487502975481e-08,
+      "logits/chosen": 2.1767947673797607,
+      "logits/rejected": 3.4365925788879395,
+      "logps/chosen": -440.6006774902344,
+      "logps/rejected": -473.36700439453125,
+      "loss": 0.3716,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 15.208317756652832,
+      "rewards/margins": 130.98887634277344,
+      "rewards/rejected": -115.78055572509766,
+      "step": 1310
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 9.540585574863127e-08,
+      "logits/chosen": 2.517340898513794,
+      "logits/rejected": 3.1486434936523438,
+      "logps/chosen": -437.44854736328125,
+      "logps/rejected": -439.794921875,
+      "loss": 0.2643,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 2.325850009918213,
+      "rewards/margins": 115.98299407958984,
+      "rewards/rejected": -113.65716552734375,
+      "step": 1320
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 9.528683646750774e-08,
+      "logits/chosen": 2.605825185775757,
+      "logits/rejected": 3.62730073928833,
+      "logps/chosen": -415.30926513671875,
+      "logps/rejected": -434.9833068847656,
+      "loss": 0.1454,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 12.735153198242188,
+      "rewards/margins": 126.29862213134766,
+      "rewards/rejected": -113.56346130371094,
+      "step": 1330
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 9.51678171863842e-08,
+      "logits/chosen": 2.488985776901245,
+      "logits/rejected": 3.484701156616211,
+      "logps/chosen": -426.5611877441406,
+      "logps/rejected": -440.25946044921875,
+      "loss": 0.5792,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 8.252630233764648,
+      "rewards/margins": 123.63459777832031,
+      "rewards/rejected": -115.3819580078125,
+      "step": 1340
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 9.504879790526065e-08,
+      "logits/chosen": 2.908905506134033,
+      "logits/rejected": 3.2668673992156982,
+      "logps/chosen": -379.87969970703125,
+      "logps/rejected": -445.2496643066406,
+      "loss": 0.3025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 7.975564479827881,
+      "rewards/margins": 144.19297790527344,
+      "rewards/rejected": -136.21739196777344,
+      "step": 1350
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 9.49297786241371e-08,
+      "logits/chosen": 2.915165424346924,
+      "logits/rejected": 3.0565438270568848,
+      "logps/chosen": -418.11669921875,
+      "logps/rejected": -470.24554443359375,
+      "loss": 0.1609,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 5.699041843414307,
+      "rewards/margins": 146.64541625976562,
+      "rewards/rejected": -140.94638061523438,
+      "step": 1360
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 9.481075934301356e-08,
+      "logits/chosen": 2.8868582248687744,
+      "logits/rejected": 3.6842854022979736,
+      "logps/chosen": -373.21185302734375,
+      "logps/rejected": -419.702392578125,
+      "loss": 0.1811,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -9.702957153320312,
+      "rewards/margins": 123.20098876953125,
+      "rewards/rejected": -132.90394592285156,
+      "step": 1370
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 9.469174006189002e-08,
+      "logits/chosen": 2.415966749191284,
+      "logits/rejected": 3.7415695190429688,
+      "logps/chosen": -477.23406982421875,
+      "logps/rejected": -459.58343505859375,
+      "loss": 0.1843,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 7.450096130371094,
+      "rewards/margins": 136.1929931640625,
+      "rewards/rejected": -128.74290466308594,
+      "step": 1380
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 9.457272078076648e-08,
+      "logits/chosen": 2.6312685012817383,
+      "logits/rejected": 3.0948596000671387,
+      "logps/chosen": -434.4181213378906,
+      "logps/rejected": -447.225341796875,
+      "loss": 0.3219,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 3.2366714477539062,
+      "rewards/margins": 148.01669311523438,
+      "rewards/rejected": -144.780029296875,
+      "step": 1390
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.445370149964293e-08,
+      "logits/chosen": 2.789755344390869,
+      "logits/rejected": 3.598536729812622,
+      "logps/chosen": -400.4031982421875,
+      "logps/rejected": -415.2806091308594,
+      "loss": 0.2459,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 13.055984497070312,
+      "rewards/margins": 139.63107299804688,
+      "rewards/rejected": -126.5750961303711,
+      "step": 1400
+    },
+    {
+      "epoch": 0.9,
+      "eval_logits/chosen": 1.8071796894073486,
+      "eval_logits/rejected": 2.5058929920196533,
+      "eval_logps/chosen": -401.0693359375,
+      "eval_logps/rejected": -428.1545715332031,
+      "eval_loss": 1.071203589439392,
+      "eval_rewards/accuracies": 0.9375,
+      "eval_rewards/chosen": 14.728099822998047,
+      "eval_rewards/margins": 144.3882293701172,
+      "eval_rewards/rejected": -129.66012573242188,
+      "eval_runtime": 76.8152,
+      "eval_samples_per_second": 13.018,
+      "eval_steps_per_second": 0.417,
+      "step": 1400
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.43346822185194e-08,
+      "logits/chosen": 2.2163331508636475,
+      "logits/rejected": 3.5338504314422607,
+      "logps/chosen": -404.0025329589844,
+      "logps/rejected": -437.22613525390625,
+      "loss": 0.2835,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 19.502986907958984,
+      "rewards/margins": 152.5010528564453,
+      "rewards/rejected": -132.998046875,
+      "step": 1410
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.421566293739586e-08,
+      "logits/chosen": 2.900045156478882,
+      "logits/rejected": 3.8935494422912598,
+      "logps/chosen": -371.714599609375,
+      "logps/rejected": -410.403564453125,
+      "loss": 0.1478,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -17.428253173828125,
+      "rewards/margins": 138.4210968017578,
+      "rewards/rejected": -155.84933471679688,
+      "step": 1420
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.409664365627231e-08,
+      "logits/chosen": 2.534538984298706,
+      "logits/rejected": 2.9124529361724854,
+      "logps/chosen": -439.07183837890625,
+      "logps/rejected": -433.35064697265625,
+      "loss": 0.2963,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 5.632425308227539,
+      "rewards/margins": 152.6386260986328,
+      "rewards/rejected": -147.00619506835938,
+      "step": 1430
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 9.397762437514877e-08,
+      "logits/chosen": 2.745168685913086,
+      "logits/rejected": 3.2941577434539795,
+      "logps/chosen": -414.8768005371094,
+      "logps/rejected": -474.46258544921875,
+      "loss": 0.1725,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.173574447631836,
+      "rewards/margins": 160.98873901367188,
+      "rewards/rejected": -178.16232299804688,
+      "step": 1440
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 9.385860509402523e-08,
+      "logits/chosen": 2.712263584136963,
+      "logits/rejected": 3.576184034347534,
+      "logps/chosen": -399.8708801269531,
+      "logps/rejected": -430.77703857421875,
+      "loss": 0.159,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -4.633294582366943,
+      "rewards/margins": 155.65737915039062,
+      "rewards/rejected": -160.29067993164062,
+      "step": 1450
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 9.373958581290168e-08,
+      "logits/chosen": 2.4558093547821045,
+      "logits/rejected": 3.536351442337036,
+      "logps/chosen": -453.64324951171875,
+      "logps/rejected": -463.8458557128906,
+      "loss": 0.166,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.3881942629814148,
+      "rewards/margins": 154.9169464111328,
+      "rewards/rejected": -154.52874755859375,
+      "step": 1460
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 9.362056653177814e-08,
+      "logits/chosen": 2.787285566329956,
+      "logits/rejected": 3.214555025100708,
+      "logps/chosen": -444.3168029785156,
+      "logps/rejected": -416.84820556640625,
+      "loss": 0.3787,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 8.255861282348633,
+      "rewards/margins": 150.24636840820312,
+      "rewards/rejected": -141.99050903320312,
+      "step": 1470
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 9.35015472506546e-08,
+      "logits/chosen": 2.6908648014068604,
+      "logits/rejected": 3.7777984142303467,
+      "logps/chosen": -413.5292053222656,
+      "logps/rejected": -414.0494079589844,
+      "loss": 0.4045,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 13.289766311645508,
+      "rewards/margins": 143.24343872070312,
+      "rewards/rejected": -129.95367431640625,
+      "step": 1480
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 9.338252796953105e-08,
+      "logits/chosen": 2.289970636367798,
+      "logits/rejected": 3.352263927459717,
+      "logps/chosen": -403.9881591796875,
+      "logps/rejected": -431.7518005371094,
+      "loss": 0.378,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 20.252124786376953,
+      "rewards/margins": 172.3944549560547,
+      "rewards/rejected": -152.142333984375,
+      "step": 1490
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 9.326350868840752e-08,
+      "logits/chosen": 3.019411563873291,
+      "logits/rejected": 3.345460891723633,
+      "logps/chosen": -434.126220703125,
+      "logps/rejected": -452.2032165527344,
+      "loss": 0.537,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 17.267078399658203,
+      "rewards/margins": 171.45938110351562,
+      "rewards/rejected": -154.19232177734375,
+      "step": 1500
+    },
+    {
+      "epoch": 0.96,
+      "eval_logits/chosen": 1.8465936183929443,
+      "eval_logits/rejected": 2.4808287620544434,
+      "eval_logps/chosen": -400.0705871582031,
+      "eval_logps/rejected": -427.70770263671875,
+      "eval_loss": 1.2171391248703003,
+      "eval_rewards/accuracies": 0.9296875,
+      "eval_rewards/chosen": 19.721826553344727,
+      "eval_rewards/margins": 147.1477813720703,
+      "eval_rewards/rejected": -127.42596435546875,
+      "eval_runtime": 76.9797,
+      "eval_samples_per_second": 12.99,
+      "eval_steps_per_second": 0.416,
+      "step": 1500
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 9.314448940728398e-08,
+      "logits/chosen": 2.212067127227783,
+      "logits/rejected": 3.642113447189331,
+      "logps/chosen": -408.46063232421875,
+      "logps/rejected": -423.57464599609375,
+      "loss": 0.1855,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 38.055782318115234,
+      "rewards/margins": 152.7517547607422,
+      "rewards/rejected": -114.69598388671875,
+      "step": 1510
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.302547012616043e-08,
+      "logits/chosen": 2.540785074234009,
+      "logits/rejected": 3.4431614875793457,
+      "logps/chosen": -421.13995361328125,
+      "logps/rejected": -475.09716796875,
+      "loss": 0.2447,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 10.023767471313477,
+      "rewards/margins": 167.34268188476562,
+      "rewards/rejected": -157.3189239501953,
+      "step": 1520
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.290645084503689e-08,
+      "logits/chosen": 2.7672924995422363,
+      "logits/rejected": 3.700913190841675,
+      "logps/chosen": -417.6861877441406,
+      "logps/rejected": -453.88818359375,
+      "loss": 0.2154,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 11.756277084350586,
+      "rewards/margins": 158.40457153320312,
+      "rewards/rejected": -146.64828491210938,
+      "step": 1530
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.278743156391336e-08,
+      "logits/chosen": 2.3169944286346436,
+      "logits/rejected": 3.3370776176452637,
+      "logps/chosen": -397.16961669921875,
+      "logps/rejected": -429.99554443359375,
+      "loss": 0.5247,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -8.517876625061035,
+      "rewards/margins": 147.62835693359375,
+      "rewards/rejected": -156.146240234375,
+      "step": 1540
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 9.26684122827898e-08,
+      "logits/chosen": 2.920701026916504,
+      "logits/rejected": 3.607771396636963,
+      "logps/chosen": -409.669189453125,
+      "logps/rejected": -427.26617431640625,
+      "loss": 0.6167,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -11.549623489379883,
+      "rewards/margins": 140.46511840820312,
+      "rewards/rejected": -152.01473999023438,
+      "step": 1550
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 9336,
+  "num_train_epochs": 6,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "trial_name": null,
+  "trial_params": null
+}