Upload folder using huggingface_hub

all_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.424075347293623,
+    "train_runtime": 2120.5143,
+    "train_samples_per_second": 3.964,
+    "train_steps_per_second": 1.321
+}

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "stabilityai/stablelm-zephyr-3b",
+  "architectures": [
+    "StableLMEpochForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_stablelm_epoch.StableLMEpochConfig",
+    "AutoModel": "modeling_stablelm_epoch.StableLMEpochForCausalLM",
+    "AutoModelForCausalLM": "stabilityai/stablelm-zephyr-3b--modeling_stablelm_epoch.StableLMEpochForCausalLM"
+  },
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "hidden_act": "silu",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 6912,
+  "max_position_embeddings": 4096,
+  "model_type": "stablelm_epoch",
+  "norm_eps": 1e-05,
+  "num_attention_heads": 32,
+  "num_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "rope_pct": 0.25,
+  "rope_theta": 10000,
+  "rotary_scaling_factor": 1.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.34.1",
+  "use_cache": false,
+  "vocab_size": 50304
+}

configuration_stablelm_epoch.py

@@ -0,0 +1,110 @@
+# coding=utf-8
+# Copyright 2023 Stability and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" StableLM Epoch model configuration"""
+from transformers import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class StableLMEpochConfig(PretrainedConfig):
+    r"""
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 50_304):
+            Vocabulary size of the StableLM model. Defines the number of different tokens that
+            can be represented by the `inputs_ids` passed when calling [`StableLMEpochModel`].
+        intermediate_size (`int`, *optional*, defaults to 6912):
+            Dimension of the MLP representations.
+        hidden_size (`int`, *optional*, defaults to 2560):
+            Dimension of the decoder layers and the pooler layer.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer decoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_key_value_heads (`int`, *optional*):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string).
+        rope_pct (`float`, *optional*, defaults to 1.0):
+            Percentage of hidden dimensions to allocate to rotary embeddings.
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        max_position_embeddings (`int`, *optional*, defaults to 2048):
+            The maximum sequence length that this model might ever be used with.
+            Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+        initializer_range (`float`, *optional*, defaults to 1e-5):
+            The standard deviation of the truncated_normal_initializer for initializing
+             all weight matrices.
+        norm_eps (`float`, *optional*, defaults to 1e-8):
+            The epsilon used by the normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions
+            (not used by all models). Only relevant if `config.is_decoder=True`.
+        tie_word_embeddings(`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+    """
+    model_type = "stablelm_epoch"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=50_304,
+        intermediate_size=6912,
+        hidden_size=2560,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=32,
+        hidden_act="silu",
+        rope_pct=0.25,
+        rope_theta=10_000,
+        max_position_embeddings=4096,
+        initializer_range=0.02,
+        norm_eps=1.0e-5,
+        use_cache=True,
+        bos_token_id=0,
+        eos_token_id=2,
+        tie_word_embeddings=False,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.intermediate_size = intermediate_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.rope_pct = rope_pct
+        self.rope_theta = rope_theta
+        self.initializer_range = initializer_range
+        self.norm_eps = norm_eps
+        self.use_cache = use_cache
+        self.tie_word_embeddings = tie_word_embeddings
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.34.1"
+}

modeling_stablelm_epoch.py

@@ -0,0 +1,687 @@
+# coding=utf-8
+# Copyright 2023 Stability AI, EleutherAI, and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This code is based off the following work:
+# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py
+# https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt_neox/modeling_gpt_neox.py
+""" PyTorch StableLM Epoch model. """
+from typing import Optional, Tuple, Union
+import math
+
+import torch
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import CrossEntropyLoss
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPast,
+    CausalLMOutputWithPast,
+)
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import logging
+from .configuration_stablelm_epoch import StableLMEpochConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+# Copied from transformers.models.bart.modeling_bart._make_causal_mask
+def _make_causal_mask(
+    input_ids_shape: torch.Size,
+    dtype: torch.dtype,
+    device: torch.device,
+    past_key_values_length: int = 0,
+):
+    """Make causal mask used for bi-directional self-attention."""
+    batch_size, tgt_len = input_ids_shape
+    mask = torch.full((tgt_len, tgt_len), torch.finfo(torch.float16).min, device=device)
+    mask_cond = torch.arange(mask.size(-1), device=device)
+    mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+    mask = mask.to(dtype)
+    if past_key_values_length > 0:
+        mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
+    return mask[None, None, :, :].expand(batch_size, 1, tgt_len, tgt_len + past_key_values_length)
+
+
+# Copied from transformers.models.bart.modeling_bart._expand_mask
+def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
+    """Expands attention_mask from `[batch_size, seq_len]` to `[batch_size, 1, tgt_seq_len, src_seq_len]`."""
+    batch_size, src_len = mask.size()
+    tgt_len = tgt_len if tgt_len is not None else src_len
+
+    expanded_mask = mask[:, None, None, :].expand(batch_size, 1, tgt_len, src_len).to(dtype)
+    inverted_mask = 1.0 - expanded_mask
+
+    return inverted_mask.masked_fill(
+        inverted_mask.to(torch.bool), torch.finfo(dtype).min
+    )
+
+
+class RotaryEmbedding(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        max_position_embeddings: int,
+        base: int = 10_000,
+        device: Optional[torch.device] = None,
+    ):
+        super().__init__()
+
+        self.dim = dim
+        self.max_position_embeddings = max_position_embeddings
+        self.base = base
+        inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, device=device, dtype=torch.float32) / self.dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+        # Build here to make `torch.jit.trace` work.
+        self._set_cos_sin_cache(
+            seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype(),
+        )
+
+    def _set_cos_sin_cache(self, seq_len: int, device: torch.device, dtype: torch.dtype):
+        self.max_seq_len_cached = seq_len
+        t = torch.arange(self.max_seq_len_cached, device=device, dtype=torch.float32)
+
+        # Don't do einsum, it converts fp32 to fp16 under AMP
+        # freqs = torch.einsum("i,j->ij", t, self.inv_freq)
+        freqs = torch.outer(t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos()[None, None, :, :].to(dtype), persistent=False)
+        self.register_buffer("sin_cached", emb.sin()[None, None, :, :].to(dtype), persistent=False)
+
+    def forward(self, x: torch.Tensor, seq_len: Optional[int] = None):
+        # x: [batch_size, num_heads, seq_len, head_size]
+        if seq_len > self.max_seq_len_cached:
+            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=torch.get_default_dtype())
+        return (
+            self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
+            self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
+        )
+
+
+def rotate_half(x: torch.Tensor):
+    """Rotates half the hidden dims of the input."""
+    x1, x2 = torch.chunk(x, 2, dim=-1)
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
+    # The first two dimensions of cos and sin are always 1, so we can `squeeze` them.
+    cos = cos.squeeze(1).squeeze(0)  # [seq_len, dim]
+    sin = sin.squeeze(1).squeeze(0)  # [seq_len, dim]
+    cos = cos[position_ids].unsqueeze(1)  # [batch_size, 1, seq_len, dim]
+    sin = sin[position_ids].unsqueeze(1)  # [batch_size, 1, seq_len, dim]
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+class MLP(nn.Module):
+    def __init__(self, config: StableLMEpochConfig):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
+        self.act_fn = nn.SiLU()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+class Attention(nn.Module):
+    def __init__(self, config: StableLMEpochConfig):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.hidden_size // self.num_heads
+        self.num_key_value_heads = config.num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.max_position_embeddings = config.max_position_embeddings
+
+        if (self.head_dim * self.num_heads) != self.hidden_size:
+            raise ValueError(
+                f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
+                f" and `num_heads`: {self.num_heads})."
+            )
+        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.o_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=False)
+
+        self._init_rope()
+
+    def _init_rope(self):
+        self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
+        self.rotary_emb = RotaryEmbedding(
+            self.rotary_ndims,
+            max_position_embeddings=self.config.max_position_embeddings,
+            base=self.config.rope_theta,
+        )
+
+    def forward(
+        self,
+        hidden_states: torch.FloatTensor,
+        attention_mask: torch.FloatTensor,
+        position_ids: torch.LongTensor,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        query_rot = query_states[..., : self.rotary_ndims]
+        query_pass = query_states[..., self.rotary_ndims :]
+        key_rot = key_states[..., : self.rotary_ndims]
+        key_pass = key_states[..., self.rotary_ndims :]
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value[0].shape[-2]
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        query_states, key_states = apply_rotary_pos_emb(query_rot, key_rot, cos, sin, position_ids)
+
+        # [batch_size, num_heads, seq_len, head_dim]
+        query_states = torch.cat((query_states, query_pass), dim=-1)
+        key_states = torch.cat((key_states, key_pass), dim=-1)
+
+        if past_key_value is not None:
+            # Reuse k, v, self_attention
+            key_states = torch.cat((past_key_value[0], key_states), dim=2)
+            value_states = torch.cat((past_key_value[1], value_states), dim=2)
+
+        past_key_value = (key_states, value_states) if use_cache else None
+
+        # Repeat k/v heads if n_kv_heads < n_heads
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
+
+        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                )
+            attn_weights = attn_weights + attention_mask
+
+        # Upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        # Merge heads
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+
+        # Final linear projection
+        attn_output = self.o_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+
+class DecoderLayer(nn.Module):
+    def __init__(self, config: StableLMEpochConfig):
+        super().__init__()
+        self.self_attn = Attention(config)
+        self.mlp = MLP(config)
+        self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps)
+        self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps)
+
+    def forward(
+        self,
+        hidden_states: Optional[torch.FloatTensor],
+        attention_mask: Optional[torch.FloatTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+    ) -> Union[Tuple[torch.Tensor], Optional[Tuple[torch.Tensor, Tuple[torch.FloatTensor, ...]]]]:
+        residual = hidden_states
+
+        hidden_states = self.input_layernorm(hidden_states)
+
+        # Self Attention
+        hidden_states, self_attn_weights, present_key_value = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_value=past_key_value,
+            output_attentions=output_attentions,
+            use_cache=use_cache,
+        )
+        hidden_states = residual + hidden_states
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (self_attn_weights,)
+
+        if use_cache:
+            outputs += (present_key_value,)
+
+        return outputs
+
+
+class StableLMEpochPreTrainedModel(PreTrainedModel):
+    """An abstract class to handle weights initialization and a simple interface
+    for downloading and loading pretrained models.
+    """
+
+    config_class = StableLMEpochConfig
+    base_model_prefix = "transformer"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["DecoderLayer"]
+    _skip_keys_device_placement = "past_key_values"
+
+    def _init_weights(self, module: nn.Module):
+        """Initialize the weights"""
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+
+    def _set_gradient_checkpointing(self, module: nn.Module, value=False):
+        if isinstance(module, StableLMEpochModel):
+            module.gradient_checkpointing = value
+
+
+class StableLMEpochModel(StableLMEpochPreTrainedModel):
+    def __init__(self, config: StableLMEpochConfig):
+        super().__init__(config)
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
+        self.layers = nn.ModuleList([DecoderLayer(config) for _ in range(config.num_hidden_layers)])
+        self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps)
+
+        self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embed_tokens
+
+    def set_input_embeddings(self, value: nn.Module):
+        self.embed_tokens = value
+
+    # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask
+    def _prepare_decoder_attention_mask(
+        self,
+        attention_mask: torch.Tensor,
+        input_shape: torch.Size,
+        inputs_embeds: torch.Tensor,
+        past_key_values_length: int,
+    ):
+        # Create causal mask
+        # [batch_size, seq_len] -> [batch_size, 1, tgt_seq_len, src_seq_len]
+        combined_attention_mask = None
+        if input_shape[-1] > 1:
+            combined_attention_mask = _make_causal_mask(
+                input_shape,
+                inputs_embeds.dtype,
+                device=inputs_embeds.device,
+                past_key_values_length=past_key_values_length,
+            )
+
+        if attention_mask is not None:
+            # [batch_size, seq_len] -> [batch_size, 1, tgt_seq_len, src_seq_len]
+            expanded_attn_mask = _expand_mask(
+                attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
+            ).to(inputs_embeds.device)
+            combined_attention_mask = expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask
+
+        return combined_attention_mask
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # Retrieve input_ids and inputs_embeds
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError(
+                "You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time"
+            )
+        elif input_ids is not None:
+            batch_size, seq_length = input_ids.shape
+        elif inputs_embeds is not None:
+            batch_size, seq_length, _ = inputs_embeds.shape
+        else:
+            raise ValueError(
+                "You have to specify either decoder_input_ids or decoder_inputs_embeds"
+            )
+
+        seq_length_with_past = seq_length
+        past_key_values_length = 0
+
+        if past_key_values is not None:
+            past_key_values_length = past_key_values[0][0].shape[2]
+            seq_length_with_past = seq_length_with_past + past_key_values_length
+
+        if position_ids is None:
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            position_ids = torch.arange(
+                past_key_values_length,
+                seq_length + past_key_values_length,
+                dtype=torch.long,
+                device=device,
+            )
+            position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
+        else:
+            position_ids = position_ids.view(-1, seq_length).long()
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+        # Embed positions
+        if attention_mask is None:
+            attention_mask = torch.ones(
+                (batch_size, seq_length_with_past),
+                dtype=torch.bool,
+                device=inputs_embeds.device,
+            )
+        attention_mask = self._prepare_decoder_attention_mask(
+            attention_mask,
+            (batch_size, seq_length),
+            inputs_embeds,
+            past_key_values_length,
+        )
+
+        hidden_states = inputs_embeds
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                )
+                use_cache = False
+
+        # Decoder layers
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+        next_decoder_cache = () if use_cache else None
+
+        for idx, decoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                all_hidden_states += (hidden_states,)
+
+            past_key_value = (
+                past_key_values[idx] if past_key_values is not None else None
+            )
+
+            if self.gradient_checkpointing and self.training:
+
+                def create_custom_forward(module):
+                    def custom_forward(*inputs):
+                        # None for past_key_value
+                        return module(*inputs, past_key_value, output_attentions)
+
+                    return custom_forward
+
+                layer_outputs = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(decoder_layer),
+                    hidden_states,
+                    attention_mask,
+                    position_ids,
+                )
+            else:
+                layer_outputs = decoder_layer(
+                    hidden_states,
+                    attention_mask=attention_mask,
+                    position_ids=position_ids,
+                    past_key_value=past_key_value,
+                    output_attentions=output_attentions,
+                    use_cache=use_cache,
+                )
+
+            hidden_states = layer_outputs[0]
+
+            if use_cache:
+                next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)
+
+            if output_attentions:
+                all_self_attns += (layer_outputs[1],)
+
+        hidden_states = self.norm(hidden_states)
+
+        # Add hidden states from the last decoder layer
+        if output_hidden_states:
+            all_hidden_states += (hidden_states,)
+
+        next_cache = next_decoder_cache if use_cache else None
+        if not return_dict:
+            return tuple(
+                v
+                for v in [hidden_states, next_cache, all_hidden_states, all_self_attns]
+                if v is not None
+            )
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=next_cache,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attns,
+        )
+
+
+class StableLMEpochForCausalLM(StableLMEpochPreTrainedModel):
+    _tied_weights_keys = ["lm_head.weight"]
+
+    def __init__(self, config: StableLMEpochConfig):
+        super().__init__(config)
+
+        self.model = StableLMEpochModel(config)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings: nn.Module):
+        self.lm_head = new_embeddings
+
+    def get_decoder(self):
+        return self.model
+
+    def set_decoder(self, decoder):
+        self.model = decoder
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.model(
+            input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states).float()
+
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+            # Enable model parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        **kwargs,
+    ):
+        # Trim decoder_input_ids if past is used
+        if past_key_values and past_key_values[0] is not None:
+            input_ids = input_ids[:, -1:]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            # Create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -1].unsqueeze(-1)
+
+        # If `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "attention_mask": attention_mask,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "position_ids": position_ids,
+            }
+        )
+        return model_inputs
+
+    @staticmethod
+    def _reorder_cache(past_key_values, beam_idx):
+        reordered_past = ()
+        for layer_past in past_key_values:
+            reordered_past += (
+                tuple(
+                    past_state.index_select(0, beam_idx.to(past_state.device))
+                    for past_state in layer_past
+                ),
+            )
+        return reordered_past
+
+
+StableLMEpochConfig.register_for_auto_class()
+StableLMEpochForCausalLM.register_for_auto_class("AutoModelForCausalLM")

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c60b9ede1bc6aae111c76a5ad9492ef1c17ac31d3034b27e14a0a80349d198c5
+size 5909512626

runs/Jan05_17-06-52_nl9mpqzac4/events.out.tfevents.1704474417.nl9mpqzac4.253.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c58ae6034bf216fa7f39abcb82ef6c0779151f76d8bc4b695d25aaac67a3d73b
+size 92791

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,215 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 2048,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}

train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.424075347293623,
+    "train_runtime": 2120.5143,
+    "train_samples_per_second": 3.964,
+    "train_steps_per_second": 1.321
+}

trainer_state.json

@@ -0,0 +1,3388 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 2802,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.999960716301628e-05,
+      "loss": 0.9669,
+      "step": 5
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.999842866441079e-05,
+      "loss": 0.8795,
+      "step": 10
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 4.9996464541220155e-05,
+      "loss": 1.1216,
+      "step": 15
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 4.999371485517079e-05,
+      "loss": 0.6686,
+      "step": 20
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 4.999017969267698e-05,
+      "loss": 0.8372,
+      "step": 25
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 4.99858591648381e-05,
+      "loss": 0.7397,
+      "step": 30
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 4.9980753407435234e-05,
+      "loss": 0.8539,
+      "step": 35
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 4.99748625809268e-05,
+      "loss": 1.0454,
+      "step": 40
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 4.9968186870443544e-05,
+      "loss": 0.7646,
+      "step": 45
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 4.9960726485782755e-05,
+      "loss": 0.8626,
+      "step": 50
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 4.995248166140163e-05,
+      "loss": 0.8773,
+      "step": 55
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 4.994345265640994e-05,
+      "loss": 0.5691,
+      "step": 60
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 4.9933639754561824e-05,
+      "loss": 0.6205,
+      "step": 65
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 4.9923043264246965e-05,
+      "loss": 0.6391,
+      "step": 70
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.9911663518480824e-05,
+      "loss": 0.5564,
+      "step": 75
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.989950087489419e-05,
+      "loss": 0.5848,
+      "step": 80
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.9886555715721964e-05,
+      "loss": 0.9124,
+      "step": 85
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.9872828447791135e-05,
+      "loss": 0.5695,
+      "step": 90
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.985831950250798e-05,
+      "loss": 0.5616,
+      "step": 95
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.9843029335844535e-05,
+      "loss": 0.6667,
+      "step": 100
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.982695842832421e-05,
+      "loss": 0.7367,
+      "step": 105
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9810107285006785e-05,
+      "loss": 0.9258,
+      "step": 110
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.979247643547242e-05,
+      "loss": 0.8087,
+      "step": 115
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.977406643380511e-05,
+      "loss": 0.8627,
+      "step": 120
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.97548778585752e-05,
+      "loss": 0.633,
+      "step": 125
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.973491131282127e-05,
+      "loss": 1.0985,
+      "step": 130
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.971416742403112e-05,
+      "loss": 0.9516,
+      "step": 135
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.969264684412208e-05,
+      "loss": 0.8711,
+      "step": 140
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.967035024942054e-05,
+      "loss": 0.7193,
+      "step": 145
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.9647278340640644e-05,
+      "loss": 0.8416,
+      "step": 150
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.9623431842862335e-05,
+      "loss": 0.6761,
+      "step": 155
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.9598811505508504e-05,
+      "loss": 0.6659,
+      "step": 160
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.957341810232147e-05,
+      "loss": 0.6969,
+      "step": 165
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.954725243133868e-05,
+      "loss": 0.8674,
+      "step": 170
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.952031531486758e-05,
+      "loss": 0.8668,
+      "step": 175
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.949260759945984e-05,
+      "loss": 0.7499,
+      "step": 180
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.946413015588466e-05,
+      "loss": 1.0306,
+      "step": 185
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.9434883879101496e-05,
+      "loss": 0.8853,
+      "step": 190
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.940486968823188e-05,
+      "loss": 0.8703,
+      "step": 195
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.937408852653055e-05,
+      "loss": 0.8031,
+      "step": 200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.934254136135581e-05,
+      "loss": 1.2864,
+      "step": 205
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.93102291841391e-05,
+      "loss": 0.6821,
+      "step": 210
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.9277153010353895e-05,
+      "loss": 0.7472,
+      "step": 215
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.9243313879483734e-05,
+      "loss": 1.2898,
+      "step": 220
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.920871285498958e-05,
+      "loss": 1.0007,
+      "step": 225
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.917335102427642e-05,
+      "loss": 0.8147,
+      "step": 230
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.913722949865902e-05,
+      "loss": 0.771,
+      "step": 235
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.91003494133271e-05,
+      "loss": 0.8844,
+      "step": 240
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.9062711927309564e-05,
+      "loss": 0.7669,
+      "step": 245
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.902431822343813e-05,
+      "loss": 1.0498,
+      "step": 250
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.898516950831015e-05,
+      "loss": 0.5262,
+      "step": 255
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.894526701225068e-05,
+      "loss": 0.8268,
+      "step": 260
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.8904611989273804e-05,
+      "loss": 1.1978,
+      "step": 265
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.8863205717043257e-05,
+      "loss": 0.7573,
+      "step": 270
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.882104949683225e-05,
+      "loss": 0.9093,
+      "step": 275
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.877814465348256e-05,
+      "loss": 0.5426,
+      "step": 280
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.873449253536295e-05,
+      "loss": 0.5552,
+      "step": 285
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.8690094514326713e-05,
+      "loss": 0.756,
+      "step": 290
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.864495198566863e-05,
+      "loss": 0.8122,
+      "step": 295
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.859906636808108e-05,
+      "loss": 0.6565,
+      "step": 300
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.855243910360948e-05,
+      "loss": 0.7606,
+      "step": 305
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.8505071657606936e-05,
+      "loss": 0.6584,
+      "step": 310
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.845696551868823e-05,
+      "loss": 0.8891,
+      "step": 315
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.840812219868299e-05,
+      "loss": 0.7888,
+      "step": 320
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.835854323258822e-05,
+      "loss": 0.804,
+      "step": 325
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.830823017852004e-05,
+      "loss": 0.6016,
+      "step": 330
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.825718461766473e-05,
+      "loss": 0.7443,
+      "step": 335
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.820540815422901e-05,
+      "loss": 0.7805,
+      "step": 340
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 4.815290241538967e-05,
+      "loss": 0.578,
+      "step": 345
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 4.809966905124238e-05,
+      "loss": 0.7461,
+      "step": 350
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 4.804570973474989e-05,
+      "loss": 0.7009,
+      "step": 355
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 4.7991026161689414e-05,
+      "loss": 0.8072,
+      "step": 360
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 4.7935620050599326e-05,
+      "loss": 0.743,
+      "step": 365
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 4.787949314272521e-05,
+      "loss": 0.6266,
+      "step": 370
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 4.782264720196506e-05,
+      "loss": 0.8777,
+      "step": 375
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 4.776508401481393e-05,
+      "loss": 0.653,
+      "step": 380
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 4.7706805390307716e-05,
+      "loss": 0.5811,
+      "step": 385
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 4.764781315996635e-05,
+      "loss": 0.5695,
+      "step": 390
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 4.75881091777362e-05,
+      "loss": 1.009,
+      "step": 395
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 4.752769531993187e-05,
+      "loss": 0.9361,
+      "step": 400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 4.746657348517716e-05,
+      "loss": 0.7177,
+      "step": 405
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 4.7404745594345455e-05,
+      "loss": 0.783,
+      "step": 410
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 4.734221359049933e-05,
+      "loss": 0.7872,
+      "step": 415
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 4.7278979438829476e-05,
+      "loss": 0.9596,
+      "step": 420
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 4.7215045126592975e-05,
+      "loss": 0.6286,
+      "step": 425
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 4.7150412663050806e-05,
+      "loss": 0.6694,
+      "step": 430
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 4.708508407940474e-05,
+      "loss": 1.1083,
+      "step": 435
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 4.701906142873348e-05,
+      "loss": 0.8941,
+      "step": 440
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 4.695234678592813e-05,
+      "loss": 0.5684,
+      "step": 445
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 4.688494224762703e-05,
+      "loss": 0.6082,
+      "step": 450
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 4.68168499321498e-05,
+      "loss": 0.6237,
+      "step": 455
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 4.674807197943084e-05,
+      "loss": 0.7826,
+      "step": 460
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 4.667861055095204e-05,
+      "loss": 0.8674,
+      "step": 465
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 4.660846782967482e-05,
+      "loss": 0.6158,
+      "step": 470
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.6537646019971606e-05,
+      "loss": 0.9251,
+      "step": 475
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.6466147347556464e-05,
+      "loss": 0.3553,
+      "step": 480
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.639397405941523e-05,
+      "loss": 0.6503,
+      "step": 485
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.632112842373487e-05,
+      "loss": 0.6233,
+      "step": 490
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.6247612729832136e-05,
+      "loss": 0.893,
+      "step": 495
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.617342928808171e-05,
+      "loss": 0.6042,
+      "step": 500
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.609858042984358e-05,
+      "loss": 0.6552,
+      "step": 505
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.602306850738968e-05,
+      "loss": 0.7058,
+      "step": 510
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5946895893830107e-05,
+      "loss": 0.59,
+      "step": 515
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.587006498303843e-05,
+      "loss": 0.7338,
+      "step": 520
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.5792578189576517e-05,
+      "loss": 0.7929,
+      "step": 525
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.5714437948618624e-05,
+      "loss": 0.5103,
+      "step": 530
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.563564671587487e-05,
+      "loss": 0.6799,
+      "step": 535
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.555620696751407e-05,
+      "loss": 1.249,
+      "step": 540
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.5476121200085934e-05,
+      "loss": 0.7596,
+      "step": 545
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.5395391930442536e-05,
+      "loss": 0.695,
+      "step": 550
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.531402169565933e-05,
+      "loss": 0.7904,
+      "step": 555
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.52320130529553e-05,
+      "loss": 0.5093,
+      "step": 560
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.51493685796127e-05,
+      "loss": 0.5815,
+      "step": 565
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 4.5066090872895944e-05,
+      "loss": 0.7107,
+      "step": 570
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 4.4982182549970105e-05,
+      "loss": 0.8838,
+      "step": 575
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 4.489764624781859e-05,
+      "loss": 0.6607,
+      "step": 580
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 4.481248462316026e-05,
+      "loss": 0.9362,
+      "step": 585
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 4.472670035236597e-05,
+      "loss": 0.7114,
+      "step": 590
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 4.4640296131374474e-05,
+      "loss": 0.8569,
+      "step": 595
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 4.4553274675607636e-05,
+      "loss": 0.8141,
+      "step": 600
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 4.446563871988517e-05,
+      "loss": 0.7217,
+      "step": 605
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 4.4377391018338624e-05,
+      "loss": 0.5951,
+      "step": 610
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 4.4288534344324884e-05,
+      "loss": 0.616,
+      "step": 615
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 4.419907149033896e-05,
+      "loss": 0.8424,
+      "step": 620
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 4.410900526792627e-05,
+      "loss": 0.6658,
+      "step": 625
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 4.401833850759428e-05,
+      "loss": 0.7002,
+      "step": 630
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.392707405872351e-05,
+      "loss": 0.6905,
+      "step": 635
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.383521478947803e-05,
+      "loss": 0.759,
+      "step": 640
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.37427635867153e-05,
+      "loss": 0.6635,
+      "step": 645
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.364972335589544e-05,
+      "loss": 0.9038,
+      "step": 650
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.355609702098995e-05,
+      "loss": 0.9281,
+      "step": 655
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.34618875243898e-05,
+      "loss": 0.6037,
+      "step": 660
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.3367097826812935e-05,
+      "loss": 0.7123,
+      "step": 665
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 4.3271730907211274e-05,
+      "loss": 0.8659,
+      "step": 670
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 4.3175789762677055e-05,
+      "loss": 0.885,
+      "step": 675
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 4.3079277408348665e-05,
+      "loss": 0.9043,
+      "step": 680
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 4.298219687731587e-05,
+      "loss": 0.6392,
+      "step": 685
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 4.2884551220524525e-05,
+      "loss": 0.9321,
+      "step": 690
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 4.278634350668065e-05,
+      "loss": 0.8969,
+      "step": 695
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 4.2687576822154e-05,
+      "loss": 0.5849,
+      "step": 700
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 4.2588254270881104e-05,
+      "loss": 0.6908,
+      "step": 705
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 4.248837897426766e-05,
+      "loss": 0.8788,
+      "step": 710
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 4.238795407109052e-05,
+      "loss": 0.7171,
+      "step": 715
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 4.228698271739894e-05,
+      "loss": 0.8489,
+      "step": 720
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.218546808641549e-05,
+      "loss": 0.7057,
+      "step": 725
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.208341336843629e-05,
+      "loss": 0.7711,
+      "step": 730
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 4.198082177073075e-05,
+      "loss": 0.7608,
+      "step": 735
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 4.1877696517440755e-05,
+      "loss": 0.6685,
+      "step": 740
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 4.17740408494794e-05,
+      "loss": 1.0127,
+      "step": 745
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 4.1669858024429085e-05,
+      "loss": 1.0633,
+      "step": 750
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 4.156515131643913e-05,
+      "loss": 0.9283,
+      "step": 755
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 4.145992401612293e-05,
+      "loss": 0.8288,
+      "step": 760
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.135417943045451e-05,
+      "loss": 0.6876,
+      "step": 765
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.12479208826646e-05,
+      "loss": 0.5607,
+      "step": 770
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.1141151712136185e-05,
+      "loss": 0.9306,
+      "step": 775
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 4.103387527429957e-05,
+      "loss": 0.8383,
+      "step": 780
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 4.092609494052695e-05,
+      "loss": 0.6689,
+      "step": 785
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 4.0817814098026424e-05,
+      "loss": 0.8616,
+      "step": 790
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 4.070903614973555e-05,
+      "loss": 0.9265,
+      "step": 795
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 4.059976451421441e-05,
+      "loss": 0.6806,
+      "step": 800
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 4.0490002625538195e-05,
+      "loss": 0.7305,
+      "step": 805
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 4.0379753933189236e-05,
+      "loss": 0.689,
+      "step": 810
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 4.026902190194864e-05,
+      "loss": 0.9028,
+      "step": 815
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 4.0157810011787376e-05,
+      "loss": 0.7079,
+      "step": 820
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 4.004612175775693e-05,
+      "loss": 0.7397,
+      "step": 825
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 3.9933960649879434e-05,
+      "loss": 0.6809,
+      "step": 830
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 3.98213302130374e-05,
+      "loss": 0.7632,
+      "step": 835
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 3.970823398686292e-05,
+      "loss": 0.8833,
+      "step": 840
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 3.959467552562642e-05,
+      "loss": 1.1581,
+      "step": 845
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 3.948065839812497e-05,
+      "loss": 0.8152,
+      "step": 850
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.936618618757012e-05,
+      "loss": 0.7035,
+      "step": 855
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9251262491475314e-05,
+      "loss": 0.6862,
+      "step": 860
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.9135890921542795e-05,
+      "loss": 0.6714,
+      "step": 865
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.902007510355014e-05,
+      "loss": 0.6129,
+      "step": 870
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.8903818677236256e-05,
+      "loss": 0.5749,
+      "step": 875
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.878712529618707e-05,
+      "loss": 0.7528,
+      "step": 880
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.866999862772063e-05,
+      "loss": 0.8081,
+      "step": 885
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.8552442352771925e-05,
+      "loss": 0.2889,
+      "step": 890
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.8434460165777145e-05,
+      "loss": 0.5895,
+      "step": 895
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.831605577455761e-05,
+      "loss": 0.7995,
+      "step": 900
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.819723290020323e-05,
+      "loss": 0.8598,
+      "step": 905
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.807799527695557e-05,
+      "loss": 0.7677,
+      "step": 910
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.79583466520905e-05,
+      "loss": 0.6112,
+      "step": 915
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.78382907858004e-05,
+      "loss": 0.701,
+      "step": 920
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.7717831451076024e-05,
+      "loss": 0.812,
+      "step": 925
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.7596972433587915e-05,
+      "loss": 0.8178,
+      "step": 930
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 0.8653,
+      "step": 935
+    },
+    {
+      "epoch": 1.01,
+      "learning_rate": 3.737843113344485e-05,
+      "loss": 0.4592,
+      "step": 940
+    },
+    {
+      "epoch": 1.01,
+      "learning_rate": 3.72564732504457e-05,
+      "loss": 0.1997,
+      "step": 945
+    },
+    {
+      "epoch": 1.02,
+      "learning_rate": 3.713413018376795e-05,
+      "loss": 0.4832,
+      "step": 950
+    },
+    {
+      "epoch": 1.02,
+      "learning_rate": 3.701140577828207e-05,
+      "loss": 0.5203,
+      "step": 955
+    },
+    {
+      "epoch": 1.03,
+      "learning_rate": 3.68883038908429e-05,
+      "loss": 0.3606,
+      "step": 960
+    },
+    {
+      "epoch": 1.03,
+      "learning_rate": 3.6764828390168374e-05,
+      "loss": 0.4231,
+      "step": 965
+    },
+    {
+      "epoch": 1.04,
+      "learning_rate": 3.664098315671793e-05,
+      "loss": 0.4086,
+      "step": 970
+    },
+    {
+      "epoch": 1.04,
+      "learning_rate": 3.651677208257063e-05,
+      "loss": 0.3105,
+      "step": 975
+    },
+    {
+      "epoch": 1.05,
+      "learning_rate": 3.639219907130276e-05,
+      "loss": 0.3642,
+      "step": 980
+    },
+    {
+      "epoch": 1.05,
+      "learning_rate": 3.626726803786519e-05,
+      "loss": 0.4502,
+      "step": 985
+    },
+    {
+      "epoch": 1.06,
+      "learning_rate": 3.6141982908460364e-05,
+      "loss": 0.4569,
+      "step": 990
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 3.601634762041887e-05,
+      "loss": 0.45,
+      "step": 995
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 3.5890366122075694e-05,
+      "loss": 0.3306,
+      "step": 1000
+    },
+    {
+      "epoch": 1.08,
+      "learning_rate": 3.576404237264621e-05,
+      "loss": 0.3047,
+      "step": 1005
+    },
+    {
+      "epoch": 1.08,
+      "learning_rate": 3.5637380342101656e-05,
+      "loss": 0.3147,
+      "step": 1010
+    },
+    {
+      "epoch": 1.09,
+      "learning_rate": 3.5510384011044436e-05,
+      "loss": 0.2347,
+      "step": 1015
+    },
+    {
+      "epoch": 1.09,
+      "learning_rate": 3.5383057370583005e-05,
+      "loss": 0.2454,
+      "step": 1020
+    },
+    {
+      "epoch": 1.1,
+      "learning_rate": 3.525540442220644e-05,
+      "loss": 0.4466,
+      "step": 1025
+    },
+    {
+      "epoch": 1.1,
+      "learning_rate": 3.512742917765866e-05,
+      "loss": 0.3689,
+      "step": 1030
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 3.499913565881241e-05,
+      "loss": 0.4169,
+      "step": 1035
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 3.487052789754279e-05,
+      "loss": 0.4434,
+      "step": 1040
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 3.4741609935600614e-05,
+      "loss": 0.2953,
+      "step": 1045
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 3.4612385824485337e-05,
+      "loss": 0.3711,
+      "step": 1050
+    },
+    {
+      "epoch": 1.13,
+      "learning_rate": 3.4482859625317766e-05,
+      "loss": 0.2047,
+      "step": 1055
+    },
+    {
+      "epoch": 1.13,
+      "learning_rate": 3.435303540871242e-05,
+      "loss": 0.204,
+      "step": 1060
+    },
+    {
+      "epoch": 1.14,
+      "learning_rate": 3.422291725464959e-05,
+      "loss": 0.3836,
+      "step": 1065
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 3.409250925234712e-05,
+      "loss": 0.2568,
+      "step": 1070
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 3.396181550013192e-05,
+      "loss": 0.4014,
+      "step": 1075
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 3.383084010531114e-05,
+      "loss": 0.4951,
+      "step": 1080
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 3.3699587184043105e-05,
+      "loss": 0.34,
+      "step": 1085
+    },
+    {
+      "epoch": 1.17,
+      "learning_rate": 3.356806086120795e-05,
+      "loss": 0.3992,
+      "step": 1090
+    },
+    {
+      "epoch": 1.17,
+      "learning_rate": 3.343626527027798e-05,
+      "loss": 0.3924,
+      "step": 1095
+    },
+    {
+      "epoch": 1.18,
+      "learning_rate": 3.3304204553187815e-05,
+      "loss": 0.1872,
+      "step": 1100
+    },
+    {
+      "epoch": 1.18,
+      "learning_rate": 3.317188286020413e-05,
+      "loss": 0.3238,
+      "step": 1105
+    },
+    {
+      "epoch": 1.19,
+      "learning_rate": 3.303930434979531e-05,
+      "loss": 0.2943,
+      "step": 1110
+    },
+    {
+      "epoch": 1.19,
+      "learning_rate": 3.290647318850074e-05,
+      "loss": 0.3754,
+      "step": 1115
+    },
+    {
+      "epoch": 1.2,
+      "learning_rate": 3.277339355079983e-05,
+      "loss": 0.3159,
+      "step": 1120
+    },
+    {
+      "epoch": 1.2,
+      "learning_rate": 3.2640069618980854e-05,
+      "loss": 0.3763,
+      "step": 1125
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 3.2506505583009516e-05,
+      "loss": 0.3898,
+      "step": 1130
+    },
+    {
+      "epoch": 1.22,
+      "learning_rate": 3.2372705640397264e-05,
+      "loss": 0.3006,
+      "step": 1135
+    },
+    {
+      "epoch": 1.22,
+      "learning_rate": 3.223867399606935e-05,
+      "loss": 0.3968,
+      "step": 1140
+    },
+    {
+      "epoch": 1.23,
+      "learning_rate": 3.210441486223274e-05,
+      "loss": 0.3107,
+      "step": 1145
+    },
+    {
+      "epoch": 1.23,
+      "learning_rate": 3.196993245824368e-05,
+      "loss": 0.4036,
+      "step": 1150
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 3.183523101047513e-05,
+      "loss": 0.3168,
+      "step": 1155
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 3.170031475218393e-05,
+      "loss": 0.2956,
+      "step": 1160
+    },
+    {
+      "epoch": 1.25,
+      "learning_rate": 3.1565187923377746e-05,
+      "loss": 0.3462,
+      "step": 1165
+    },
+    {
+      "epoch": 1.25,
+      "learning_rate": 3.142985477068185e-05,
+      "loss": 0.3532,
+      "step": 1170
+    },
+    {
+      "epoch": 1.26,
+      "learning_rate": 3.129431954720565e-05,
+      "loss": 0.3928,
+      "step": 1175
+    },
+    {
+      "epoch": 1.26,
+      "learning_rate": 3.1158586512409e-05,
+      "loss": 0.4801,
+      "step": 1180
+    },
+    {
+      "epoch": 1.27,
+      "learning_rate": 3.1022659931968395e-05,
+      "loss": 0.4299,
+      "step": 1185
+    },
+    {
+      "epoch": 1.27,
+      "learning_rate": 3.0886544077642865e-05,
+      "loss": 0.4258,
+      "step": 1190
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 3.075024322713972e-05,
+      "loss": 0.5112,
+      "step": 1195
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 3.0613761663980184e-05,
+      "loss": 0.3401,
+      "step": 1200
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 3.0477103677364694e-05,
+      "loss": 0.4322,
+      "step": 1205
+    },
+    {
+      "epoch": 1.3,
+      "learning_rate": 3.0340273562038146e-05,
+      "loss": 0.4011,
+      "step": 1210
+    },
+    {
+      "epoch": 1.3,
+      "learning_rate": 3.0203275618154935e-05,
+      "loss": 0.3879,
+      "step": 1215
+    },
+    {
+      "epoch": 1.31,
+      "learning_rate": 3.0066114151143775e-05,
+      "loss": 0.3586,
+      "step": 1220
+    },
+    {
+      "epoch": 1.31,
+      "learning_rate": 2.9928793471572432e-05,
+      "loss": 0.3994,
+      "step": 1225
+    },
+    {
+      "epoch": 1.32,
+      "learning_rate": 2.9791317895012234e-05,
+      "loss": 0.2322,
+      "step": 1230
+    },
+    {
+      "epoch": 1.32,
+      "learning_rate": 2.965369174190243e-05,
+      "loss": 0.3513,
+      "step": 1235
+    },
+    {
+      "epoch": 1.33,
+      "learning_rate": 2.9515919337414472e-05,
+      "loss": 0.307,
+      "step": 1240
+    },
+    {
+      "epoch": 1.33,
+      "learning_rate": 2.937800501131601e-05,
+      "loss": 0.3364,
+      "step": 1245
+    },
+    {
+      "epoch": 1.34,
+      "learning_rate": 2.9239953097834876e-05,
+      "loss": 0.3243,
+      "step": 1250
+    },
+    {
+      "epoch": 1.34,
+      "learning_rate": 2.910176793552287e-05,
+      "loss": 0.4326,
+      "step": 1255
+    },
+    {
+      "epoch": 1.35,
+      "learning_rate": 2.8963453867119354e-05,
+      "loss": 0.3193,
+      "step": 1260
+    },
+    {
+      "epoch": 1.35,
+      "learning_rate": 2.8825015239414856e-05,
+      "loss": 0.3131,
+      "step": 1265
+    },
+    {
+      "epoch": 1.36,
+      "learning_rate": 2.8686456403114415e-05,
+      "loss": 0.3406,
+      "step": 1270
+    },
+    {
+      "epoch": 1.37,
+      "learning_rate": 2.854778171270085e-05,
+      "loss": 0.3749,
+      "step": 1275
+    },
+    {
+      "epoch": 1.37,
+      "learning_rate": 2.8408995526297926e-05,
+      "loss": 0.334,
+      "step": 1280
+    },
+    {
+      "epoch": 1.38,
+      "learning_rate": 2.8270102205533406e-05,
+      "loss": 0.3929,
+      "step": 1285
+    },
+    {
+      "epoch": 1.38,
+      "learning_rate": 2.8131106115401933e-05,
+      "loss": 0.3972,
+      "step": 1290
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 2.7992011624127888e-05,
+      "loss": 0.2296,
+      "step": 1295
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 2.7852823103028116e-05,
+      "loss": 0.3938,
+      "step": 1300
+    },
+    {
+      "epoch": 1.4,
+      "learning_rate": 2.771354492637451e-05,
+      "loss": 0.3469,
+      "step": 1305
+    },
+    {
+      "epoch": 1.4,
+      "learning_rate": 2.7574181471256578e-05,
+      "loss": 0.3049,
+      "step": 1310
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 2.743473711744387e-05,
+      "loss": 0.3872,
+      "step": 1315
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 2.7295216247248327e-05,
+      "loss": 0.2893,
+      "step": 1320
+    },
+    {
+      "epoch": 1.42,
+      "learning_rate": 2.7155623245386584e-05,
+      "loss": 0.3781,
+      "step": 1325
+    },
+    {
+      "epoch": 1.42,
+      "learning_rate": 2.701596249884214e-05,
+      "loss": 0.3755,
+      "step": 1330
+    },
+    {
+      "epoch": 1.43,
+      "learning_rate": 2.6876238396727504e-05,
+      "loss": 0.2317,
+      "step": 1335
+    },
+    {
+      "epoch": 1.43,
+      "learning_rate": 2.6736455330146278e-05,
+      "loss": 0.5266,
+      "step": 1340
+    },
+    {
+      "epoch": 1.44,
+      "learning_rate": 2.6596617692055105e-05,
+      "loss": 0.367,
+      "step": 1345
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 2.6456729877125663e-05,
+      "loss": 0.3744,
+      "step": 1350
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 2.631679628160655e-05,
+      "loss": 0.4376,
+      "step": 1355
+    },
+    {
+      "epoch": 1.46,
+      "learning_rate": 2.6176821303185066e-05,
+      "loss": 0.4219,
+      "step": 1360
+    },
+    {
+      "epoch": 1.46,
+      "learning_rate": 2.6036809340849106e-05,
+      "loss": 0.2541,
+      "step": 1365
+    },
+    {
+      "epoch": 1.47,
+      "learning_rate": 2.5896764794748813e-05,
+      "loss": 0.3705,
+      "step": 1370
+    },
+    {
+      "epoch": 1.47,
+      "learning_rate": 2.5756692066058346e-05,
+      "loss": 0.4607,
+      "step": 1375
+    },
+    {
+      "epoch": 1.48,
+      "learning_rate": 2.5616595556837573e-05,
+      "loss": 0.5631,
+      "step": 1380
+    },
+    {
+      "epoch": 1.48,
+      "learning_rate": 2.547647966989371e-05,
+      "loss": 0.5518,
+      "step": 1385
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 2.533634880864293e-05,
+      "loss": 0.3245,
+      "step": 1390
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 2.519620737697204e-05,
+      "loss": 0.3764,
+      "step": 1395
+    },
+    {
+      "epoch": 1.5,
+      "learning_rate": 2.5056059779100017e-05,
+      "loss": 0.3896,
+      "step": 1400
+    },
+    {
+      "epoch": 1.5,
+      "learning_rate": 2.4915910419439627e-05,
+      "loss": 0.2681,
+      "step": 1405
+    },
+    {
+      "epoch": 1.51,
+      "learning_rate": 2.4775763702459026e-05,
+      "loss": 0.3076,
+      "step": 1410
+    },
+    {
+      "epoch": 1.51,
+      "learning_rate": 2.463562403254327e-05,
+      "loss": 0.4762,
+      "step": 1415
+    },
+    {
+      "epoch": 1.52,
+      "learning_rate": 2.4495495813855994e-05,
+      "loss": 0.3402,
+      "step": 1420
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 2.4355383450200957e-05,
+      "loss": 0.3426,
+      "step": 1425
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 2.421529134488359e-05,
+      "loss": 0.323,
+      "step": 1430
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 2.407522390057272e-05,
+      "loss": 0.2517,
+      "step": 1435
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 2.3935185519162133e-05,
+      "loss": 0.2985,
+      "step": 1440
+    },
+    {
+      "epoch": 1.55,
+      "learning_rate": 2.3795180601632257e-05,
+      "loss": 0.377,
+      "step": 1445
+    },
+    {
+      "epoch": 1.55,
+      "learning_rate": 2.3655213547911846e-05,
+      "loss": 0.3976,
+      "step": 1450
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 2.3515288756739732e-05,
+      "loss": 0.5024,
+      "step": 1455
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 2.3375410625526527e-05,
+      "loss": 0.3413,
+      "step": 1460
+    },
+    {
+      "epoch": 1.57,
+      "learning_rate": 2.3235583550216507e-05,
+      "loss": 0.2962,
+      "step": 1465
+    },
+    {
+      "epoch": 1.57,
+      "learning_rate": 2.309581192514937e-05,
+      "loss": 0.3109,
+      "step": 1470
+    },
+    {
+      "epoch": 1.58,
+      "learning_rate": 2.295610014292221e-05,
+      "loss": 0.313,
+      "step": 1475
+    },
+    {
+      "epoch": 1.58,
+      "learning_rate": 2.2816452594251454e-05,
+      "loss": 0.4659,
+      "step": 1480
+    },
+    {
+      "epoch": 1.59,
+      "learning_rate": 2.2676873667834822e-05,
+      "loss": 0.3142,
+      "step": 1485
+    },
+    {
+      "epoch": 1.6,
+      "learning_rate": 2.253736775021349e-05,
+      "loss": 0.4029,
+      "step": 1490
+    },
+    {
+      "epoch": 1.6,
+      "learning_rate": 2.239793922563415e-05,
+      "loss": 0.4598,
+      "step": 1495
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 2.22585924759113e-05,
+      "loss": 0.3665,
+      "step": 1500
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 2.2119331880289482e-05,
+      "loss": 0.3814,
+      "step": 1505
+    },
+    {
+      "epoch": 1.62,
+      "learning_rate": 2.1980161815305685e-05,
+      "loss": 0.3147,
+      "step": 1510
+    },
+    {
+      "epoch": 1.62,
+      "learning_rate": 2.1841086654651787e-05,
+      "loss": 0.4053,
+      "step": 1515
+    },
+    {
+      "epoch": 1.63,
+      "learning_rate": 2.1702110769037138e-05,
+      "loss": 0.3571,
+      "step": 1520
+    },
+    {
+      "epoch": 1.63,
+      "learning_rate": 2.1563238526051128e-05,
+      "loss": 0.4268,
+      "step": 1525
+    },
+    {
+      "epoch": 1.64,
+      "learning_rate": 2.1424474290026002e-05,
+      "loss": 0.2449,
+      "step": 1530
+    },
+    {
+      "epoch": 1.64,
+      "learning_rate": 2.128582242189971e-05,
+      "loss": 0.3293,
+      "step": 1535
+    },
+    {
+      "epoch": 1.65,
+      "learning_rate": 2.114728727907875e-05,
+      "loss": 0.3671,
+      "step": 1540
+    },
+    {
+      "epoch": 1.65,
+      "learning_rate": 2.1008873215301346e-05,
+      "loss": 0.3639,
+      "step": 1545
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 2.0870584580500555e-05,
+      "loss": 0.328,
+      "step": 1550
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 2.0732425720667605e-05,
+      "loss": 0.3223,
+      "step": 1555
+    },
+    {
+      "epoch": 1.67,
+      "learning_rate": 2.0594400977715268e-05,
+      "loss": 0.4182,
+      "step": 1560
+    },
+    {
+      "epoch": 1.68,
+      "learning_rate": 2.045651468934145e-05,
+      "loss": 0.4449,
+      "step": 1565
+    },
+    {
+      "epoch": 1.68,
+      "learning_rate": 2.0318771188892823e-05,
+      "loss": 0.368,
+      "step": 1570
+    },
+    {
+      "epoch": 1.69,
+      "learning_rate": 2.018117480522871e-05,
+      "loss": 0.237,
+      "step": 1575
+    },
+    {
+      "epoch": 1.69,
+      "learning_rate": 2.0043729862584952e-05,
+      "loss": 0.2892,
+      "step": 1580
+    },
+    {
+      "epoch": 1.7,
+      "learning_rate": 1.99064406804381e-05,
+      "loss": 0.3522,
+      "step": 1585
+    },
+    {
+      "epoch": 1.7,
+      "learning_rate": 1.9769311573369613e-05,
+      "loss": 0.4601,
+      "step": 1590
+    },
+    {
+      "epoch": 1.71,
+      "learning_rate": 1.9632346850930265e-05,
+      "loss": 0.3318,
+      "step": 1595
+    },
+    {
+      "epoch": 1.71,
+      "learning_rate": 1.9495550817504742e-05,
+      "loss": 0.3333,
+      "step": 1600
+    },
+    {
+      "epoch": 1.72,
+      "learning_rate": 1.935892777217633e-05,
+      "loss": 0.3585,
+      "step": 1605
+    },
+    {
+      "epoch": 1.72,
+      "learning_rate": 1.922248200859183e-05,
+      "loss": 0.2718,
+      "step": 1610
+    },
+    {
+      "epoch": 1.73,
+      "learning_rate": 1.908621781482662e-05,
+      "loss": 0.2824,
+      "step": 1615
+    },
+    {
+      "epoch": 1.73,
+      "learning_rate": 1.89501394732499e-05,
+      "loss": 0.2374,
+      "step": 1620
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 1.8814251260390067e-05,
+      "loss": 0.3704,
+      "step": 1625
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 1.8678557446800403e-05,
+      "loss": 0.3356,
+      "step": 1630
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 1.854306229692476e-05,
+      "loss": 0.295,
+      "step": 1635
+    },
+    {
+      "epoch": 1.76,
+      "learning_rate": 1.8407770068963615e-05,
+      "loss": 0.3668,
+      "step": 1640
+    },
+    {
+      "epoch": 1.76,
+      "learning_rate": 1.8272685014740258e-05,
+      "loss": 0.3668,
+      "step": 1645
+    },
+    {
+      "epoch": 1.77,
+      "learning_rate": 1.8137811379567076e-05,
+      "loss": 0.2927,
+      "step": 1650
+    },
+    {
+      "epoch": 1.77,
+      "learning_rate": 1.8003153402112248e-05,
+      "loss": 0.3767,
+      "step": 1655
+    },
+    {
+      "epoch": 1.78,
+      "learning_rate": 1.7868715314266464e-05,
+      "loss": 0.3051,
+      "step": 1660
+    },
+    {
+      "epoch": 1.78,
+      "learning_rate": 1.773450134100997e-05,
+      "loss": 0.2894,
+      "step": 1665
+    },
+    {
+      "epoch": 1.79,
+      "learning_rate": 1.760051570027974e-05,
+      "loss": 0.329,
+      "step": 1670
+    },
+    {
+      "epoch": 1.79,
+      "learning_rate": 1.746676260283699e-05,
+      "loss": 0.3993,
+      "step": 1675
+    },
+    {
+      "epoch": 1.8,
+      "learning_rate": 1.7333246252134767e-05,
+      "loss": 0.2806,
+      "step": 1680
+    },
+    {
+      "epoch": 1.8,
+      "learning_rate": 1.7199970844185943e-05,
+      "loss": 0.359,
+      "step": 1685
+    },
+    {
+      "epoch": 1.81,
+      "learning_rate": 1.706694056743122e-05,
+      "loss": 0.2649,
+      "step": 1690
+    },
+    {
+      "epoch": 1.81,
+      "learning_rate": 1.693415960260764e-05,
+      "loss": 0.3227,
+      "step": 1695
+    },
+    {
+      "epoch": 1.82,
+      "learning_rate": 1.6801632122617095e-05,
+      "loss": 0.2917,
+      "step": 1700
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 1.6669362292395214e-05,
+      "loss": 0.4163,
+      "step": 1705
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 1.6537354268780498e-05,
+      "loss": 0.4411,
+      "step": 1710
+    },
+    {
+      "epoch": 1.84,
+      "learning_rate": 1.6405612200383645e-05,
+      "loss": 0.2494,
+      "step": 1715
+    },
+    {
+      "epoch": 1.84,
+      "learning_rate": 1.6274140227457213e-05,
+      "loss": 0.5221,
+      "step": 1720
+    },
+    {
+      "epoch": 1.85,
+      "learning_rate": 1.6142942481765448e-05,
+      "loss": 0.3146,
+      "step": 1725
+    },
+    {
+      "epoch": 1.85,
+      "learning_rate": 1.6012023086454503e-05,
+      "loss": 0.3994,
+      "step": 1730
+    },
+    {
+      "epoch": 1.86,
+      "learning_rate": 1.588138615592278e-05,
+      "loss": 0.2006,
+      "step": 1735
+    },
+    {
+      "epoch": 1.86,
+      "learning_rate": 1.5751035795691727e-05,
+      "loss": 0.5552,
+      "step": 1740
+    },
+    {
+      "epoch": 1.87,
+      "learning_rate": 1.5620976102276714e-05,
+      "loss": 0.3602,
+      "step": 1745
+    },
+    {
+      "epoch": 1.87,
+      "learning_rate": 1.5491211163058357e-05,
+      "loss": 0.2798,
+      "step": 1750
+    },
+    {
+      "epoch": 1.88,
+      "learning_rate": 1.5361745056154048e-05,
+      "loss": 0.2709,
+      "step": 1755
+    },
+    {
+      "epoch": 1.88,
+      "learning_rate": 1.523258185028977e-05,
+      "loss": 0.3803,
+      "step": 1760
+    },
+    {
+      "epoch": 1.89,
+      "learning_rate": 1.5103725604672275e-05,
+      "loss": 0.2861,
+      "step": 1765
+    },
+    {
+      "epoch": 1.9,
+      "learning_rate": 1.4975180368861458e-05,
+      "loss": 0.3521,
+      "step": 1770
+    },
+    {
+      "epoch": 1.9,
+      "learning_rate": 1.4846950182643143e-05,
+      "loss": 0.3457,
+      "step": 1775
+    },
+    {
+      "epoch": 1.91,
+      "learning_rate": 1.4719039075902091e-05,
+      "loss": 0.333,
+      "step": 1780
+    },
+    {
+      "epoch": 1.91,
+      "learning_rate": 1.4591451068495382e-05,
+      "loss": 0.2831,
+      "step": 1785
+    },
+    {
+      "epoch": 1.92,
+      "learning_rate": 1.4464190170126034e-05,
+      "loss": 0.3263,
+      "step": 1790
+    },
+    {
+      "epoch": 1.92,
+      "learning_rate": 1.433726038021707e-05,
+      "loss": 0.301,
+      "step": 1795
+    },
+    {
+      "epoch": 1.93,
+      "learning_rate": 1.4210665687785734e-05,
+      "loss": 0.2941,
+      "step": 1800
+    },
+    {
+      "epoch": 1.93,
+      "learning_rate": 1.4084410071318201e-05,
+      "loss": 0.2209,
+      "step": 1805
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 1.3958497498644529e-05,
+      "loss": 0.4193,
+      "step": 1810
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 1.3832931926813907e-05,
+      "loss": 0.302,
+      "step": 1815
+    },
+    {
+      "epoch": 1.95,
+      "learning_rate": 1.3707717301970416e-05,
+      "loss": 0.3154,
+      "step": 1820
+    },
+    {
+      "epoch": 1.95,
+      "learning_rate": 1.3582857559228867e-05,
+      "loss": 0.548,
+      "step": 1825
+    },
+    {
+      "epoch": 1.96,
+      "learning_rate": 1.345835662255126e-05,
+      "loss": 0.2957,
+      "step": 1830
+    },
+    {
+      "epoch": 1.96,
+      "learning_rate": 1.3334218404623373e-05,
+      "loss": 0.2932,
+      "step": 1835
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 1.3210446806731857e-05,
+      "loss": 0.3821,
+      "step": 1840
+    },
+    {
+      "epoch": 1.98,
+      "learning_rate": 1.308704571864161e-05,
+      "loss": 0.3919,
+      "step": 1845
+    },
+    {
+      "epoch": 1.98,
+      "learning_rate": 1.2964019018473545e-05,
+      "loss": 0.2465,
+      "step": 1850
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 1.2841370572582661e-05,
+      "loss": 0.4155,
+      "step": 1855
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 1.2719104235436613e-05,
+      "loss": 0.379,
+      "step": 1860
+    },
+    {
+      "epoch": 2.0,
+      "learning_rate": 1.2597223849494538e-05,
+      "loss": 0.304,
+      "step": 1865
+    },
+    {
+      "epoch": 2.0,
+      "learning_rate": 1.2475733245086263e-05,
+      "loss": 0.3512,
+      "step": 1870
+    },
+    {
+      "epoch": 2.01,
+      "learning_rate": 1.2354636240292031e-05,
+      "loss": 0.1898,
+      "step": 1875
+    },
+    {
+      "epoch": 2.01,
+      "learning_rate": 1.2233936640822385e-05,
+      "loss": 0.1608,
+      "step": 1880
+    },
+    {
+      "epoch": 2.02,
+      "learning_rate": 1.211363823989867e-05,
+      "loss": 0.1453,
+      "step": 1885
+    },
+    {
+      "epoch": 2.02,
+      "learning_rate": 1.1993744818133742e-05,
+      "loss": 0.1639,
+      "step": 1890
+    },
+    {
+      "epoch": 2.03,
+      "learning_rate": 1.187426014341323e-05,
+      "loss": 0.1394,
+      "step": 1895
+    },
+    {
+      "epoch": 2.03,
+      "learning_rate": 1.1755187970777065e-05,
+      "loss": 0.1596,
+      "step": 1900
+    },
+    {
+      "epoch": 2.04,
+      "learning_rate": 1.1636532042301512e-05,
+      "loss": 0.1185,
+      "step": 1905
+    },
+    {
+      "epoch": 2.04,
+      "learning_rate": 1.1518296086981514e-05,
+      "loss": 0.1359,
+      "step": 1910
+    },
+    {
+      "epoch": 2.05,
+      "learning_rate": 1.1400483820613563e-05,
+      "loss": 0.1435,
+      "step": 1915
+    },
+    {
+      "epoch": 2.06,
+      "learning_rate": 1.1283098945678902e-05,
+      "loss": 0.1491,
+      "step": 1920
+    },
+    {
+      "epoch": 2.06,
+      "learning_rate": 1.1166145151227117e-05,
+      "loss": 0.1697,
+      "step": 1925
+    },
+    {
+      "epoch": 2.07,
+      "learning_rate": 1.1049626112760314e-05,
+      "loss": 0.1224,
+      "step": 1930
+    },
+    {
+      "epoch": 2.07,
+      "learning_rate": 1.0933545492117473e-05,
+      "loss": 0.1467,
+      "step": 1935
+    },
+    {
+      "epoch": 2.08,
+      "learning_rate": 1.0817906937359482e-05,
+      "loss": 0.1813,
+      "step": 1940
+    },
+    {
+      "epoch": 2.08,
+      "learning_rate": 1.070271408265441e-05,
+      "loss": 0.1128,
+      "step": 1945
+    },
+    {
+      "epoch": 2.09,
+      "learning_rate": 1.058797054816335e-05,
+      "loss": 0.1896,
+      "step": 1950
+    },
+    {
+      "epoch": 2.09,
+      "learning_rate": 1.0473679939926626e-05,
+      "loss": 0.1205,
+      "step": 1955
+    },
+    {
+      "epoch": 2.1,
+      "learning_rate": 1.0359845849750466e-05,
+      "loss": 0.1581,
+      "step": 1960
+    },
+    {
+      "epoch": 2.1,
+      "learning_rate": 1.0246471855094106e-05,
+      "loss": 0.1542,
+      "step": 1965
+    },
+    {
+      "epoch": 2.11,
+      "learning_rate": 1.0133561518957402e-05,
+      "loss": 0.1542,
+      "step": 1970
+    },
+    {
+      "epoch": 2.11,
+      "learning_rate": 1.0021118389768833e-05,
+      "loss": 0.0885,
+      "step": 1975
+    },
+    {
+      "epoch": 2.12,
+      "learning_rate": 9.909146001273947e-06,
+      "loss": 0.109,
+      "step": 1980
+    },
+    {
+      "epoch": 2.13,
+      "learning_rate": 9.797647872424413e-06,
+      "loss": 0.1327,
+      "step": 1985
+    },
+    {
+      "epoch": 2.13,
+      "learning_rate": 9.686627507267287e-06,
+      "loss": 0.1584,
+      "step": 1990
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 9.576088394835023e-06,
+      "loss": 0.2896,
+      "step": 1995
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 9.466034009035724e-06,
+      "loss": 0.2332,
+      "step": 2000
+    },
+    {
+      "epoch": 2.15,
+      "learning_rate": 9.356467808544033e-06,
+      "loss": 0.1039,
+      "step": 2005
+    },
+    {
+      "epoch": 2.15,
+      "learning_rate": 9.247393236692412e-06,
+      "loss": 0.1014,
+      "step": 2010
+    },
+    {
+      "epoch": 2.16,
+      "learning_rate": 9.13881372136293e-06,
+      "loss": 0.0737,
+      "step": 2015
+    },
+    {
+      "epoch": 2.16,
+      "learning_rate": 9.030732674879514e-06,
+      "loss": 0.1649,
+      "step": 2020
+    },
+    {
+      "epoch": 2.17,
+      "learning_rate": 8.923153493900757e-06,
+      "loss": 0.1502,
+      "step": 2025
+    },
+    {
+      "epoch": 2.17,
+      "learning_rate": 8.816079559313147e-06,
+      "loss": 0.2198,
+      "step": 2030
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 8.709514236124783e-06,
+      "loss": 0.2312,
+      "step": 2035
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 8.603460873359687e-06,
+      "loss": 0.1627,
+      "step": 2040
+    },
+    {
+      "epoch": 2.19,
+      "learning_rate": 8.49792280395251e-06,
+      "loss": 0.0986,
+      "step": 2045
+    },
+    {
+      "epoch": 2.19,
+      "learning_rate": 8.392903344643807e-06,
+      "loss": 0.1889,
+      "step": 2050
+    },
+    {
+      "epoch": 2.2,
+      "learning_rate": 8.288405795875773e-06,
+      "loss": 0.1543,
+      "step": 2055
+    },
+    {
+      "epoch": 2.21,
+      "learning_rate": 8.184433441688564e-06,
+      "loss": 0.0997,
+      "step": 2060
+    },
+    {
+      "epoch": 2.21,
+      "learning_rate": 8.08098954961706e-06,
+      "loss": 0.1241,
+      "step": 2065
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 7.978077370588196e-06,
+      "loss": 0.1259,
+      "step": 2070
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 7.875700138818756e-06,
+      "loss": 0.1803,
+      "step": 2075
+    },
+    {
+      "epoch": 2.23,
+      "learning_rate": 7.773861071713779e-06,
+      "loss": 0.1437,
+      "step": 2080
+    },
+    {
+      "epoch": 2.23,
+      "learning_rate": 7.672563369765429e-06,
+      "loss": 0.131,
+      "step": 2085
+    },
+    {
+      "epoch": 2.24,
+      "learning_rate": 7.571810216452388e-06,
+      "loss": 0.193,
+      "step": 2090
+    },
+    {
+      "epoch": 2.24,
+      "learning_rate": 7.4716047781398485e-06,
+      "loss": 0.1658,
+      "step": 2095
+    },
+    {
+      "epoch": 2.25,
+      "learning_rate": 7.3719502039799856e-06,
+      "loss": 0.1377,
+      "step": 2100
+    },
+    {
+      "epoch": 2.25,
+      "learning_rate": 7.2728496258129915e-06,
+      "loss": 0.1433,
+      "step": 2105
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 7.174306158068625e-06,
+      "loss": 0.1309,
+      "step": 2110
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 7.0763228976683885e-06,
+      "loss": 0.149,
+      "step": 2115
+    },
+    {
+      "epoch": 2.27,
+      "learning_rate": 6.9789029239281574e-06,
+      "loss": 0.1573,
+      "step": 2120
+    },
+    {
+      "epoch": 2.28,
+      "learning_rate": 6.8820492984614324e-06,
+      "loss": 0.1268,
+      "step": 2125
+    },
+    {
+      "epoch": 2.28,
+      "learning_rate": 6.785765065083083e-06,
+      "loss": 0.0947,
+      "step": 2130
+    },
+    {
+      "epoch": 2.29,
+      "learning_rate": 6.690053249713743e-06,
+      "loss": 0.1492,
+      "step": 2135
+    },
+    {
+      "epoch": 2.29,
+      "learning_rate": 6.594916860284692e-06,
+      "loss": 0.1955,
+      "step": 2140
+    },
+    {
+      "epoch": 2.3,
+      "learning_rate": 6.50035888664329e-06,
+      "loss": 0.1137,
+      "step": 2145
+    },
+    {
+      "epoch": 2.3,
+      "learning_rate": 6.406382300459079e-06,
+      "loss": 0.1055,
+      "step": 2150
+    },
+    {
+      "epoch": 2.31,
+      "learning_rate": 6.312990055130355e-06,
+      "loss": 0.1119,
+      "step": 2155
+    },
+    {
+      "epoch": 2.31,
+      "learning_rate": 6.22018508569136e-06,
+      "loss": 0.1138,
+      "step": 2160
+    },
+    {
+      "epoch": 2.32,
+      "learning_rate": 6.1279703087200186e-06,
+      "loss": 0.1401,
+      "step": 2165
+    },
+    {
+      "epoch": 2.32,
+      "learning_rate": 6.036348622246327e-06,
+      "loss": 0.1442,
+      "step": 2170
+    },
+    {
+      "epoch": 2.33,
+      "learning_rate": 5.945322905661244e-06,
+      "loss": 0.115,
+      "step": 2175
+    },
+    {
+      "epoch": 2.33,
+      "learning_rate": 5.854896019626208e-06,
+      "loss": 0.1394,
+      "step": 2180
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 5.765070805983219e-06,
+      "loss": 0.1419,
+      "step": 2185
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 5.675850087665563e-06,
+      "loss": 0.1352,
+      "step": 2190
+    },
+    {
+      "epoch": 2.35,
+      "learning_rate": 5.58723666860908e-06,
+      "loss": 0.1157,
+      "step": 2195
+    },
+    {
+      "epoch": 2.36,
+      "learning_rate": 5.4992333336640115e-06,
+      "loss": 0.1597,
+      "step": 2200
+    },
+    {
+      "epoch": 2.36,
+      "learning_rate": 5.411842848507542e-06,
+      "loss": 0.1668,
+      "step": 2205
+    },
+    {
+      "epoch": 2.37,
+      "learning_rate": 5.325067959556834e-06,
+      "loss": 0.106,
+      "step": 2210
+    },
+    {
+      "epoch": 2.37,
+      "learning_rate": 5.238911393882751e-06,
+      "loss": 0.1626,
+      "step": 2215
+    },
+    {
+      "epoch": 2.38,
+      "learning_rate": 5.1533758591241075e-06,
+      "loss": 0.1001,
+      "step": 2220
+    },
+    {
+      "epoch": 2.38,
+      "learning_rate": 5.068464043402632e-06,
+      "loss": 0.1199,
+      "step": 2225
+    },
+    {
+      "epoch": 2.39,
+      "learning_rate": 4.984178615238436e-06,
+      "loss": 0.1118,
+      "step": 2230
+    },
+    {
+      "epoch": 2.39,
+      "learning_rate": 4.900522223466208e-06,
+      "loss": 0.2024,
+      "step": 2235
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 4.8174974971519075e-06,
+      "loss": 0.0992,
+      "step": 2240
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 4.735107045510179e-06,
+      "loss": 0.0686,
+      "step": 2245
+    },
+    {
+      "epoch": 2.41,
+      "learning_rate": 4.653353457822349e-06,
+      "loss": 0.2271,
+      "step": 2250
+    },
+    {
+      "epoch": 2.41,
+      "learning_rate": 4.572239303355033e-06,
+      "loss": 0.1831,
+      "step": 2255
+    },
+    {
+      "epoch": 2.42,
+      "learning_rate": 4.491767131279414e-06,
+      "loss": 0.1541,
+      "step": 2260
+    },
+    {
+      "epoch": 2.43,
+      "learning_rate": 4.411939470591125e-06,
+      "loss": 0.2611,
+      "step": 2265
+    },
+    {
+      "epoch": 2.43,
+      "learning_rate": 4.332758830030767e-06,
+      "loss": 0.1216,
+      "step": 2270
+    },
+    {
+      "epoch": 2.44,
+      "learning_rate": 4.254227698005048e-06,
+      "loss": 0.112,
+      "step": 2275
+    },
+    {
+      "epoch": 2.44,
+      "learning_rate": 4.176348542508621e-06,
+      "loss": 0.1712,
+      "step": 2280
+    },
+    {
+      "epoch": 2.45,
+      "learning_rate": 4.099123811046471e-06,
+      "loss": 0.1919,
+      "step": 2285
+    },
+    {
+      "epoch": 2.45,
+      "learning_rate": 4.0225559305570676e-06,
+      "loss": 0.1018,
+      "step": 2290
+    },
+    {
+      "epoch": 2.46,
+      "learning_rate": 3.946647307336013e-06,
+      "loss": 0.1054,
+      "step": 2295
+    },
+    {
+      "epoch": 2.46,
+      "learning_rate": 3.871400326960481e-06,
+      "loss": 0.1212,
+      "step": 2300
+    },
+    {
+      "epoch": 2.47,
+      "learning_rate": 3.7968173542142187e-06,
+      "loss": 0.1063,
+      "step": 2305
+    },
+    {
+      "epoch": 2.47,
+      "learning_rate": 3.722900733013221e-06,
+      "loss": 0.1183,
+      "step": 2310
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 3.6496527863320916e-06,
+      "loss": 0.3089,
+      "step": 2315
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 3.5770758161310288e-06,
+      "loss": 0.1188,
+      "step": 2320
+    },
+    {
+      "epoch": 2.49,
+      "learning_rate": 3.505172103283483e-06,
+      "loss": 0.1374,
+      "step": 2325
+    },
+    {
+      "epoch": 2.49,
+      "learning_rate": 3.4339439075044555e-06,
+      "loss": 0.2563,
+      "step": 2330
+    },
+    {
+      "epoch": 2.5,
+      "learning_rate": 3.3633934672795242e-06,
+      "loss": 0.0739,
+      "step": 2335
+    },
+    {
+      "epoch": 2.51,
+      "learning_rate": 3.293522999794443e-06,
+      "loss": 0.1082,
+      "step": 2340
+    },
+    {
+      "epoch": 2.51,
+      "learning_rate": 3.2243347008655333e-06,
+      "loss": 0.1598,
+      "step": 2345
+    },
+    {
+      "epoch": 2.52,
+      "learning_rate": 3.1558307448705886e-06,
+      "loss": 0.1337,
+      "step": 2350
+    },
+    {
+      "epoch": 2.52,
+      "learning_rate": 3.0880132846806103e-06,
+      "loss": 0.1227,
+      "step": 2355
+    },
+    {
+      "epoch": 2.53,
+      "learning_rate": 3.020884451592126e-06,
+      "loss": 0.1446,
+      "step": 2360
+    },
+    {
+      "epoch": 2.53,
+      "learning_rate": 2.9544463552601875e-06,
+      "loss": 0.1262,
+      "step": 2365
+    },
+    {
+      "epoch": 2.54,
+      "learning_rate": 2.8887010836321087e-06,
+      "loss": 0.131,
+      "step": 2370
+    },
+    {
+      "epoch": 2.54,
+      "learning_rate": 2.8236507028818306e-06,
+      "loss": 0.3064,
+      "step": 2375
+    },
+    {
+      "epoch": 2.55,
+      "learning_rate": 2.759297257344981e-06,
+      "loss": 0.1329,
+      "step": 2380
+    },
+    {
+      "epoch": 2.55,
+      "learning_rate": 2.695642769454629e-06,
+      "loss": 0.1269,
+      "step": 2385
+    },
+    {
+      "epoch": 2.56,
+      "learning_rate": 2.6326892396777465e-06,
+      "loss": 0.1515,
+      "step": 2390
+    },
+    {
+      "epoch": 2.56,
+      "learning_rate": 2.5704386464522946e-06,
+      "loss": 0.1497,
+      "step": 2395
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 2.508892946125119e-06,
+      "loss": 0.1247,
+      "step": 2400
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 2.4480540728903876e-06,
+      "loss": 0.1079,
+      "step": 2405
+    },
+    {
+      "epoch": 2.58,
+      "learning_rate": 2.3879239387288615e-06,
+      "loss": 0.0815,
+      "step": 2410
+    },
+    {
+      "epoch": 2.59,
+      "learning_rate": 2.3285044333477834e-06,
+      "loss": 0.0905,
+      "step": 2415
+    },
+    {
+      "epoch": 2.59,
+      "learning_rate": 2.269797424121492e-06,
+      "loss": 0.1124,
+      "step": 2420
+    },
+    {
+      "epoch": 2.6,
+      "learning_rate": 2.2118047560327425e-06,
+      "loss": 0.1781,
+      "step": 2425
+    },
+    {
+      "epoch": 2.6,
+      "learning_rate": 2.154528251614721e-06,
+      "loss": 0.2224,
+      "step": 2430
+    },
+    {
+      "epoch": 2.61,
+      "learning_rate": 2.0979697108937685e-06,
+      "loss": 0.1721,
+      "step": 2435
+    },
+    {
+      "epoch": 2.61,
+      "learning_rate": 2.0421309113328042e-06,
+      "loss": 0.1502,
+      "step": 2440
+    },
+    {
+      "epoch": 2.62,
+      "learning_rate": 1.9870136077754787e-06,
+      "loss": 0.1646,
+      "step": 2445
+    },
+    {
+      "epoch": 2.62,
+      "learning_rate": 1.9326195323910082e-06,
+      "loss": 0.1088,
+      "step": 2450
+    },
+    {
+      "epoch": 2.63,
+      "learning_rate": 1.8789503946197579e-06,
+      "loss": 0.1143,
+      "step": 2455
+    },
+    {
+      "epoch": 2.63,
+      "learning_rate": 1.8260078811195041e-06,
+      "loss": 0.1692,
+      "step": 2460
+    },
+    {
+      "epoch": 2.64,
+      "learning_rate": 1.7737936557124301e-06,
+      "loss": 0.1727,
+      "step": 2465
+    },
+    {
+      "epoch": 2.64,
+      "learning_rate": 1.7223093593328494e-06,
+      "loss": 0.0885,
+      "step": 2470
+    },
+    {
+      "epoch": 2.65,
+      "learning_rate": 1.6715566099756024e-06,
+      "loss": 0.0786,
+      "step": 2475
+    },
+    {
+      "epoch": 2.66,
+      "learning_rate": 1.6215370026452598e-06,
+      "loss": 0.1128,
+      "step": 2480
+    },
+    {
+      "epoch": 2.66,
+      "learning_rate": 1.5722521093059496e-06,
+      "loss": 0.1089,
+      "step": 2485
+    },
+    {
+      "epoch": 2.67,
+      "learning_rate": 1.5237034788319837e-06,
+      "loss": 0.1772,
+      "step": 2490
+    },
+    {
+      "epoch": 2.67,
+      "learning_rate": 1.4758926369591614e-06,
+      "loss": 0.1795,
+      "step": 2495
+    },
+    {
+      "epoch": 2.68,
+      "learning_rate": 1.4288210862368395e-06,
+      "loss": 0.1264,
+      "step": 2500
+    },
+    {
+      "epoch": 2.68,
+      "learning_rate": 1.3824903059806937e-06,
+      "loss": 0.1129,
+      "step": 2505
+    },
+    {
+      "epoch": 2.69,
+      "learning_rate": 1.3369017522262438e-06,
+      "loss": 0.0854,
+      "step": 2510
+    },
+    {
+      "epoch": 2.69,
+      "learning_rate": 1.2920568576830882e-06,
+      "loss": 0.0878,
+      "step": 2515
+    },
+    {
+      "epoch": 2.7,
+      "learning_rate": 1.2479570316898726e-06,
+      "loss": 0.1346,
+      "step": 2520
+    },
+    {
+      "epoch": 2.7,
+      "learning_rate": 1.2046036601700146e-06,
+      "loss": 0.1298,
+      "step": 2525
+    },
+    {
+      "epoch": 2.71,
+      "learning_rate": 1.1619981055881162e-06,
+      "loss": 0.1598,
+      "step": 2530
+    },
+    {
+      "epoch": 2.71,
+      "learning_rate": 1.1201417069071934e-06,
+      "loss": 0.1295,
+      "step": 2535
+    },
+    {
+      "epoch": 2.72,
+      "learning_rate": 1.0790357795465527e-06,
+      "loss": 0.127,
+      "step": 2540
+    },
+    {
+      "epoch": 2.72,
+      "learning_rate": 1.038681615340481e-06,
+      "loss": 0.1714,
+      "step": 2545
+    },
+    {
+      "epoch": 2.73,
+      "learning_rate": 9.99080482497622e-07,
+      "loss": 0.1494,
+      "step": 2550
+    },
+    {
+      "epoch": 2.74,
+      "learning_rate": 9.602336255611522e-07,
+      "loss": 0.0671,
+      "step": 2555
+    },
+    {
+      "epoch": 2.74,
+      "learning_rate": 9.221422653696299e-07,
+      "loss": 0.1641,
+      "step": 2560
+    },
+    {
+      "epoch": 2.75,
+      "learning_rate": 8.848075990186639e-07,
+      "loss": 0.1617,
+      "step": 2565
+    },
+    {
+      "epoch": 2.75,
+      "learning_rate": 8.482307998232686e-07,
+      "loss": 0.1436,
+      "step": 2570
+    },
+    {
+      "epoch": 2.76,
+      "learning_rate": 8.12413017281008e-07,
+      "loss": 0.1328,
+      "step": 2575
+    },
+    {
+      "epoch": 2.76,
+      "learning_rate": 7.773553770358488e-07,
+      "loss": 0.1612,
+      "step": 2580
+    },
+    {
+      "epoch": 2.77,
+      "learning_rate": 7.430589808428062e-07,
+      "loss": 0.141,
+      "step": 2585
+    },
+    {
+      "epoch": 2.77,
+      "learning_rate": 7.095249065333016e-07,
+      "loss": 0.1324,
+      "step": 2590
+    },
+    {
+      "epoch": 2.78,
+      "learning_rate": 6.767542079813089e-07,
+      "loss": 0.2075,
+      "step": 2595
+    },
+    {
+      "epoch": 2.78,
+      "learning_rate": 6.447479150702207e-07,
+      "loss": 0.1804,
+      "step": 2600
+    },
+    {
+      "epoch": 2.79,
+      "learning_rate": 6.135070336604737e-07,
+      "loss": 0.184,
+      "step": 2605
+    },
+    {
+      "epoch": 2.79,
+      "learning_rate": 5.830325455579627e-07,
+      "loss": 0.2522,
+      "step": 2610
+    },
+    {
+      "epoch": 2.8,
+      "learning_rate": 5.533254084831657e-07,
+      "loss": 0.1122,
+      "step": 2615
+    },
+    {
+      "epoch": 2.81,
+      "learning_rate": 5.24386556041051e-07,
+      "loss": 0.1075,
+      "step": 2620
+    },
+    {
+      "epoch": 2.81,
+      "learning_rate": 4.962168976917397e-07,
+      "loss": 0.1535,
+      "step": 2625
+    },
+    {
+      "epoch": 2.82,
+      "learning_rate": 4.688173187219258e-07,
+      "loss": 0.1201,
+      "step": 2630
+    },
+    {
+      "epoch": 2.82,
+      "learning_rate": 4.4218868021703996e-07,
+      "loss": 0.0932,
+      "step": 2635
+    },
+    {
+      "epoch": 2.83,
+      "learning_rate": 4.163318190342075e-07,
+      "loss": 0.1969,
+      "step": 2640
+    },
+    {
+      "epoch": 2.83,
+      "learning_rate": 3.912475477759386e-07,
+      "loss": 0.1997,
+      "step": 2645
+    },
+    {
+      "epoch": 2.84,
+      "learning_rate": 3.6693665476458526e-07,
+      "loss": 0.1271,
+      "step": 2650
+    },
+    {
+      "epoch": 2.84,
+      "learning_rate": 3.433999040175828e-07,
+      "loss": 0.166,
+      "step": 2655
+    },
+    {
+      "epoch": 2.85,
+      "learning_rate": 3.206380352234195e-07,
+      "loss": 0.1474,
+      "step": 2660
+    },
+    {
+      "epoch": 2.85,
+      "learning_rate": 2.986517637184133e-07,
+      "loss": 0.1127,
+      "step": 2665
+    },
+    {
+      "epoch": 2.86,
+      "learning_rate": 2.774417804642021e-07,
+      "loss": 0.1357,
+      "step": 2670
+    },
+    {
+      "epoch": 2.86,
+      "learning_rate": 2.570087520260611e-07,
+      "loss": 0.0642,
+      "step": 2675
+    },
+    {
+      "epoch": 2.87,
+      "learning_rate": 2.3735332055192515e-07,
+      "loss": 0.111,
+      "step": 2680
+    },
+    {
+      "epoch": 2.87,
+      "learning_rate": 2.184761037522326e-07,
+      "loss": 0.1453,
+      "step": 2685
+    },
+    {
+      "epoch": 2.88,
+      "learning_rate": 2.0037769488049363e-07,
+      "loss": 0.2098,
+      "step": 2690
+    },
+    {
+      "epoch": 2.89,
+      "learning_rate": 1.8305866271465521e-07,
+      "loss": 0.1603,
+      "step": 2695
+    },
+    {
+      "epoch": 2.89,
+      "learning_rate": 1.665195515392265e-07,
+      "loss": 0.1162,
+      "step": 2700
+    },
+    {
+      "epoch": 2.9,
+      "learning_rate": 1.50760881128173e-07,
+      "loss": 0.1069,
+      "step": 2705
+    },
+    {
+      "epoch": 2.9,
+      "learning_rate": 1.3578314672857972e-07,
+      "loss": 0.1596,
+      "step": 2710
+    },
+    {
+      "epoch": 2.91,
+      "learning_rate": 1.2158681904508306e-07,
+      "loss": 0.1448,
+      "step": 2715
+    },
+    {
+      "epoch": 2.91,
+      "learning_rate": 1.0817234422508815e-07,
+      "loss": 0.099,
+      "step": 2720
+    },
+    {
+      "epoch": 2.92,
+      "learning_rate": 9.554014384474119e-08,
+      "loss": 0.1657,
+      "step": 2725
+    },
+    {
+      "epoch": 2.92,
+      "learning_rate": 8.369061489568453e-08,
+      "loss": 0.1751,
+      "step": 2730
+    },
+    {
+      "epoch": 2.93,
+      "learning_rate": 7.262412977257215e-08,
+      "loss": 0.1825,
+      "step": 2735
+    },
+    {
+      "epoch": 2.93,
+      "learning_rate": 6.234103626137355e-08,
+      "loss": 0.2671,
+      "step": 2740
+    },
+    {
+      "epoch": 2.94,
+      "learning_rate": 5.284165752844905e-08,
+      "loss": 0.138,
+      "step": 2745
+    },
+    {
+      "epoch": 2.94,
+      "learning_rate": 4.412629211037744e-08,
+      "loss": 0.1538,
+      "step": 2750
+    },
+    {
+      "epoch": 2.95,
+      "learning_rate": 3.6195213904588465e-08,
+      "loss": 0.0941,
+      "step": 2755
+    },
+    {
+      "epoch": 2.96,
+      "learning_rate": 2.9048672160750246e-08,
+      "loss": 0.1369,
+      "step": 2760
+    },
+    {
+      "epoch": 2.96,
+      "learning_rate": 2.2686891472933903e-08,
+      "loss": 0.1327,
+      "step": 2765
+    },
+    {
+      "epoch": 2.97,
+      "learning_rate": 1.7110071772560853e-08,
+      "loss": 0.0837,
+      "step": 2770
+    },
+    {
+      "epoch": 2.97,
+      "learning_rate": 1.231838832211063e-08,
+      "loss": 0.2136,
+      "step": 2775
+    },
+    {
+      "epoch": 2.98,
+      "learning_rate": 8.311991709619716e-09,
+      "loss": 0.1371,
+      "step": 2780
+    },
+    {
+      "epoch": 2.98,
+      "learning_rate": 5.091007843954776e-09,
+      "loss": 0.1605,
+      "step": 2785
+    },
+    {
+      "epoch": 2.99,
+      "learning_rate": 2.655537950838061e-09,
+      "loss": 0.1396,
+      "step": 2790
+    },
+    {
+      "epoch": 2.99,
+      "learning_rate": 1.00565856968049e-09,
+      "loss": 0.0865,
+      "step": 2795
+    },
+    {
+      "epoch": 3.0,
+      "learning_rate": 1.414215511780226e-10,
+      "loss": 0.1582,
+      "step": 2800
+    },
+    {
+      "epoch": 3.0,
+      "step": 2802,
+      "total_flos": 9.781740939681792e+16,
+      "train_loss": 0.424075347293623,
+      "train_runtime": 2120.5143,
+      "train_samples_per_second": 3.964,
+      "train_steps_per_second": 1.321
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 2802,
+  "num_train_epochs": 3,
+  "save_steps": 5000,
+  "total_flos": 9.781740939681792e+16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3697516be7f89e5ccee07f68b65fdf1fe5a338155ff85dc776525de25b402dc8
+size 4728