Hugging Face's logo

theblackcat102
/
pythia-12b-deduped-sft

Text Generation
Transformers
PyTorch
English
gpt_neox
sft
text-generation-inference
Inference Endpoints
Model card Files Community
theblackcat102 commited on
Commit
afd3781
•
1 Parent(s): 532af29

Upload 6 files

Browse files
Files changed (6) hide show
  1. config.json +25 -0
  2. pytorch_model.bin.index.json +551 -0
  3. special_tokens_map.json +13 -0
  4. tokenizer.json +0 -0
  5. tokenizer_config.json +10 -0
  6. trainer_state.json +1804 -0
config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "EleutherAI/pythia-12b-deduped",
3
+ "architectures": [
4
+ "GPTNeoXForCausalLM"
5
+ ],
6
+ "bos_token_id": 0,
7
+ "eos_token_id": 0,
8
+ "hidden_act": "gelu",
9
+ "hidden_size": 5120,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 20480,
12
+ "layer_norm_eps": 1e-05,
13
+ "max_position_embeddings": 2048,
14
+ "model_type": "gpt_neox",
15
+ "num_attention_heads": 40,
16
+ "num_hidden_layers": 36,
17
+ "rotary_emb_base": 10000,
18
+ "rotary_pct": 0.25,
19
+ "tie_word_embeddings": false,
20
+ "torch_dtype": "float16",
21
+ "transformers_version": "4.25.1",
22
+ "use_cache": true,
23
+ "use_parallel_residual": true,
24
+ "vocab_size": 50281
25
+ }
pytorch_model.bin.index.json ADDED
@@ -0,0 +1,551 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 23834805448
4
+ },
5
+ "weight_map": {
6
+ "embed_out.weight": "pytorch_model-00003-of-00003.bin",
7
+ "gpt_neox.embed_in.weight": "pytorch_model-00001-of-00003.bin",
8
+ "gpt_neox.final_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
9
+ "gpt_neox.final_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
10
+ "gpt_neox.layers.0.attention.bias": "pytorch_model-00001-of-00003.bin",
11
+ "gpt_neox.layers.0.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
12
+ "gpt_neox.layers.0.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
13
+ "gpt_neox.layers.0.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
14
+ "gpt_neox.layers.0.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
15
+ "gpt_neox.layers.0.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
16
+ "gpt_neox.layers.0.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
17
+ "gpt_neox.layers.0.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
18
+ "gpt_neox.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
19
+ "gpt_neox.layers.0.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
20
+ "gpt_neox.layers.0.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
21
+ "gpt_neox.layers.0.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
22
+ "gpt_neox.layers.0.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
23
+ "gpt_neox.layers.0.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
24
+ "gpt_neox.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
25
+ "gpt_neox.layers.1.attention.bias": "pytorch_model-00001-of-00003.bin",
26
+ "gpt_neox.layers.1.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
27
+ "gpt_neox.layers.1.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
28
+ "gpt_neox.layers.1.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
29
+ "gpt_neox.layers.1.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
30
+ "gpt_neox.layers.1.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
31
+ "gpt_neox.layers.1.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
32
+ "gpt_neox.layers.1.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
33
+ "gpt_neox.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
34
+ "gpt_neox.layers.1.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
35
+ "gpt_neox.layers.1.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
36
+ "gpt_neox.layers.1.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
37
+ "gpt_neox.layers.1.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
38
+ "gpt_neox.layers.1.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
39
+ "gpt_neox.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
40
+ "gpt_neox.layers.10.attention.bias": "pytorch_model-00001-of-00003.bin",
41
+ "gpt_neox.layers.10.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
42
+ "gpt_neox.layers.10.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
43
+ "gpt_neox.layers.10.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
44
+ "gpt_neox.layers.10.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
45
+ "gpt_neox.layers.10.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
46
+ "gpt_neox.layers.10.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
47
+ "gpt_neox.layers.10.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
48
+ "gpt_neox.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
49
+ "gpt_neox.layers.10.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
50
+ "gpt_neox.layers.10.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
51
+ "gpt_neox.layers.10.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
52
+ "gpt_neox.layers.10.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
53
+ "gpt_neox.layers.10.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
54
+ "gpt_neox.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
55
+ "gpt_neox.layers.11.attention.bias": "pytorch_model-00001-of-00003.bin",
56
+ "gpt_neox.layers.11.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
57
+ "gpt_neox.layers.11.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
58
+ "gpt_neox.layers.11.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
59
+ "gpt_neox.layers.11.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
60
+ "gpt_neox.layers.11.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
61
+ "gpt_neox.layers.11.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
62
+ "gpt_neox.layers.11.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
63
+ "gpt_neox.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
64
+ "gpt_neox.layers.11.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
65
+ "gpt_neox.layers.11.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
66
+ "gpt_neox.layers.11.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
67
+ "gpt_neox.layers.11.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
68
+ "gpt_neox.layers.11.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
69
+ "gpt_neox.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
70
+ "gpt_neox.layers.12.attention.bias": "pytorch_model-00001-of-00003.bin",
71
+ "gpt_neox.layers.12.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
72
+ "gpt_neox.layers.12.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
73
+ "gpt_neox.layers.12.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
74
+ "gpt_neox.layers.12.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
75
+ "gpt_neox.layers.12.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
76
+ "gpt_neox.layers.12.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
77
+ "gpt_neox.layers.12.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
78
+ "gpt_neox.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
79
+ "gpt_neox.layers.12.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
80
+ "gpt_neox.layers.12.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
81
+ "gpt_neox.layers.12.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
82
+ "gpt_neox.layers.12.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
83
+ "gpt_neox.layers.12.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
84
+ "gpt_neox.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
85
+ "gpt_neox.layers.13.attention.bias": "pytorch_model-00001-of-00003.bin",
86
+ "gpt_neox.layers.13.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
87
+ "gpt_neox.layers.13.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
88
+ "gpt_neox.layers.13.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
89
+ "gpt_neox.layers.13.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
90
+ "gpt_neox.layers.13.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
91
+ "gpt_neox.layers.13.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
92
+ "gpt_neox.layers.13.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
93
+ "gpt_neox.layers.13.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
94
+ "gpt_neox.layers.13.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
95
+ "gpt_neox.layers.13.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
96
+ "gpt_neox.layers.13.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
97
+ "gpt_neox.layers.13.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
98
+ "gpt_neox.layers.13.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
99
+ "gpt_neox.layers.13.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
100
+ "gpt_neox.layers.14.attention.bias": "pytorch_model-00001-of-00003.bin",
101
+ "gpt_neox.layers.14.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
102
+ "gpt_neox.layers.14.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
103
+ "gpt_neox.layers.14.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
104
+ "gpt_neox.layers.14.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
105
+ "gpt_neox.layers.14.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
106
+ "gpt_neox.layers.14.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
107
+ "gpt_neox.layers.14.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
108
+ "gpt_neox.layers.14.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
109
+ "gpt_neox.layers.14.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
110
+ "gpt_neox.layers.14.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
111
+ "gpt_neox.layers.14.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
112
+ "gpt_neox.layers.14.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
113
+ "gpt_neox.layers.14.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
114
+ "gpt_neox.layers.14.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
115
+ "gpt_neox.layers.15.attention.bias": "pytorch_model-00002-of-00003.bin",
116
+ "gpt_neox.layers.15.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
117
+ "gpt_neox.layers.15.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
118
+ "gpt_neox.layers.15.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
119
+ "gpt_neox.layers.15.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
120
+ "gpt_neox.layers.15.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
121
+ "gpt_neox.layers.15.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
122
+ "gpt_neox.layers.15.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
123
+ "gpt_neox.layers.15.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
124
+ "gpt_neox.layers.15.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
125
+ "gpt_neox.layers.15.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
126
+ "gpt_neox.layers.15.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
127
+ "gpt_neox.layers.15.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
128
+ "gpt_neox.layers.15.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
129
+ "gpt_neox.layers.15.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
130
+ "gpt_neox.layers.16.attention.bias": "pytorch_model-00002-of-00003.bin",
131
+ "gpt_neox.layers.16.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
132
+ "gpt_neox.layers.16.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
133
+ "gpt_neox.layers.16.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
134
+ "gpt_neox.layers.16.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
135
+ "gpt_neox.layers.16.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
136
+ "gpt_neox.layers.16.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
137
+ "gpt_neox.layers.16.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
138
+ "gpt_neox.layers.16.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
139
+ "gpt_neox.layers.16.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
140
+ "gpt_neox.layers.16.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
141
+ "gpt_neox.layers.16.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
142
+ "gpt_neox.layers.16.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
143
+ "gpt_neox.layers.16.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
144
+ "gpt_neox.layers.16.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
145
+ "gpt_neox.layers.17.attention.bias": "pytorch_model-00002-of-00003.bin",
146
+ "gpt_neox.layers.17.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
147
+ "gpt_neox.layers.17.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
148
+ "gpt_neox.layers.17.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
149
+ "gpt_neox.layers.17.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
150
+ "gpt_neox.layers.17.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
151
+ "gpt_neox.layers.17.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
152
+ "gpt_neox.layers.17.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
153
+ "gpt_neox.layers.17.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
154
+ "gpt_neox.layers.17.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
155
+ "gpt_neox.layers.17.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
156
+ "gpt_neox.layers.17.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
157
+ "gpt_neox.layers.17.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
158
+ "gpt_neox.layers.17.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
159
+ "gpt_neox.layers.17.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
160
+ "gpt_neox.layers.18.attention.bias": "pytorch_model-00002-of-00003.bin",
161
+ "gpt_neox.layers.18.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
162
+ "gpt_neox.layers.18.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
163
+ "gpt_neox.layers.18.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
164
+ "gpt_neox.layers.18.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
165
+ "gpt_neox.layers.18.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
166
+ "gpt_neox.layers.18.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
167
+ "gpt_neox.layers.18.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
168
+ "gpt_neox.layers.18.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
169
+ "gpt_neox.layers.18.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
170
+ "gpt_neox.layers.18.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
171
+ "gpt_neox.layers.18.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
172
+ "gpt_neox.layers.18.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
173
+ "gpt_neox.layers.18.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
174
+ "gpt_neox.layers.18.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
175
+ "gpt_neox.layers.19.attention.bias": "pytorch_model-00002-of-00003.bin",
176
+ "gpt_neox.layers.19.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
177
+ "gpt_neox.layers.19.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
178
+ "gpt_neox.layers.19.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
179
+ "gpt_neox.layers.19.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
180
+ "gpt_neox.layers.19.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
181
+ "gpt_neox.layers.19.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
182
+ "gpt_neox.layers.19.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
183
+ "gpt_neox.layers.19.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
184
+ "gpt_neox.layers.19.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
185
+ "gpt_neox.layers.19.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
186
+ "gpt_neox.layers.19.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
187
+ "gpt_neox.layers.19.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
188
+ "gpt_neox.layers.19.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
189
+ "gpt_neox.layers.19.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
190
+ "gpt_neox.layers.2.attention.bias": "pytorch_model-00001-of-00003.bin",
191
+ "gpt_neox.layers.2.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
192
+ "gpt_neox.layers.2.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
193
+ "gpt_neox.layers.2.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
194
+ "gpt_neox.layers.2.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
195
+ "gpt_neox.layers.2.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
196
+ "gpt_neox.layers.2.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
197
+ "gpt_neox.layers.2.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
198
+ "gpt_neox.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
199
+ "gpt_neox.layers.2.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
200
+ "gpt_neox.layers.2.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
201
+ "gpt_neox.layers.2.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
202
+ "gpt_neox.layers.2.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
203
+ "gpt_neox.layers.2.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
204
+ "gpt_neox.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
205
+ "gpt_neox.layers.20.attention.bias": "pytorch_model-00002-of-00003.bin",
206
+ "gpt_neox.layers.20.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
207
+ "gpt_neox.layers.20.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
208
+ "gpt_neox.layers.20.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
209
+ "gpt_neox.layers.20.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
210
+ "gpt_neox.layers.20.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
211
+ "gpt_neox.layers.20.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
212
+ "gpt_neox.layers.20.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
213
+ "gpt_neox.layers.20.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
214
+ "gpt_neox.layers.20.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
215
+ "gpt_neox.layers.20.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
216
+ "gpt_neox.layers.20.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
217
+ "gpt_neox.layers.20.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
218
+ "gpt_neox.layers.20.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
219
+ "gpt_neox.layers.20.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
220
+ "gpt_neox.layers.21.attention.bias": "pytorch_model-00002-of-00003.bin",
221
+ "gpt_neox.layers.21.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
222
+ "gpt_neox.layers.21.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
223
+ "gpt_neox.layers.21.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
224
+ "gpt_neox.layers.21.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
225
+ "gpt_neox.layers.21.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
226
+ "gpt_neox.layers.21.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
227
+ "gpt_neox.layers.21.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
228
+ "gpt_neox.layers.21.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
229
+ "gpt_neox.layers.21.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
230
+ "gpt_neox.layers.21.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
231
+ "gpt_neox.layers.21.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
232
+ "gpt_neox.layers.21.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
233
+ "gpt_neox.layers.21.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
234
+ "gpt_neox.layers.21.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
235
+ "gpt_neox.layers.22.attention.bias": "pytorch_model-00002-of-00003.bin",
236
+ "gpt_neox.layers.22.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
237
+ "gpt_neox.layers.22.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
238
+ "gpt_neox.layers.22.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
239
+ "gpt_neox.layers.22.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
240
+ "gpt_neox.layers.22.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
241
+ "gpt_neox.layers.22.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
242
+ "gpt_neox.layers.22.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
243
+ "gpt_neox.layers.22.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
244
+ "gpt_neox.layers.22.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
245
+ "gpt_neox.layers.22.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
246
+ "gpt_neox.layers.22.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
247
+ "gpt_neox.layers.22.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
248
+ "gpt_neox.layers.22.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
249
+ "gpt_neox.layers.22.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
250
+ "gpt_neox.layers.23.attention.bias": "pytorch_model-00002-of-00003.bin",
251
+ "gpt_neox.layers.23.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
252
+ "gpt_neox.layers.23.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
253
+ "gpt_neox.layers.23.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
254
+ "gpt_neox.layers.23.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
255
+ "gpt_neox.layers.23.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
256
+ "gpt_neox.layers.23.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
257
+ "gpt_neox.layers.23.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
258
+ "gpt_neox.layers.23.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
259
+ "gpt_neox.layers.23.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
260
+ "gpt_neox.layers.23.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
261
+ "gpt_neox.layers.23.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
262
+ "gpt_neox.layers.23.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
263
+ "gpt_neox.layers.23.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
264
+ "gpt_neox.layers.23.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
265
+ "gpt_neox.layers.24.attention.bias": "pytorch_model-00002-of-00003.bin",
266
+ "gpt_neox.layers.24.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
267
+ "gpt_neox.layers.24.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
268
+ "gpt_neox.layers.24.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
269
+ "gpt_neox.layers.24.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
270
+ "gpt_neox.layers.24.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
271
+ "gpt_neox.layers.24.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
272
+ "gpt_neox.layers.24.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
273
+ "gpt_neox.layers.24.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
274
+ "gpt_neox.layers.24.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
275
+ "gpt_neox.layers.24.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
276
+ "gpt_neox.layers.24.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
277
+ "gpt_neox.layers.24.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
278
+ "gpt_neox.layers.24.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
279
+ "gpt_neox.layers.24.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
280
+ "gpt_neox.layers.25.attention.bias": "pytorch_model-00002-of-00003.bin",
281
+ "gpt_neox.layers.25.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
282
+ "gpt_neox.layers.25.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
283
+ "gpt_neox.layers.25.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
284
+ "gpt_neox.layers.25.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
285
+ "gpt_neox.layers.25.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
286
+ "gpt_neox.layers.25.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
287
+ "gpt_neox.layers.25.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
288
+ "gpt_neox.layers.25.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
289
+ "gpt_neox.layers.25.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
290
+ "gpt_neox.layers.25.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
291
+ "gpt_neox.layers.25.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
292
+ "gpt_neox.layers.25.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
293
+ "gpt_neox.layers.25.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
294
+ "gpt_neox.layers.25.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
295
+ "gpt_neox.layers.26.attention.bias": "pytorch_model-00002-of-00003.bin",
296
+ "gpt_neox.layers.26.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
297
+ "gpt_neox.layers.26.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
298
+ "gpt_neox.layers.26.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
299
+ "gpt_neox.layers.26.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
300
+ "gpt_neox.layers.26.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
301
+ "gpt_neox.layers.26.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
302
+ "gpt_neox.layers.26.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
303
+ "gpt_neox.layers.26.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
304
+ "gpt_neox.layers.26.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
305
+ "gpt_neox.layers.26.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
306
+ "gpt_neox.layers.26.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
307
+ "gpt_neox.layers.26.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
308
+ "gpt_neox.layers.26.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
309
+ "gpt_neox.layers.26.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
310
+ "gpt_neox.layers.27.attention.bias": "pytorch_model-00002-of-00003.bin",
311
+ "gpt_neox.layers.27.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
312
+ "gpt_neox.layers.27.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
313
+ "gpt_neox.layers.27.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
314
+ "gpt_neox.layers.27.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
315
+ "gpt_neox.layers.27.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
316
+ "gpt_neox.layers.27.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
317
+ "gpt_neox.layers.27.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
318
+ "gpt_neox.layers.27.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
319
+ "gpt_neox.layers.27.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
320
+ "gpt_neox.layers.27.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
321
+ "gpt_neox.layers.27.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
322
+ "gpt_neox.layers.27.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
323
+ "gpt_neox.layers.27.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
324
+ "gpt_neox.layers.27.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
325
+ "gpt_neox.layers.28.attention.bias": "pytorch_model-00002-of-00003.bin",
326
+ "gpt_neox.layers.28.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
327
+ "gpt_neox.layers.28.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
328
+ "gpt_neox.layers.28.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
329
+ "gpt_neox.layers.28.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
330
+ "gpt_neox.layers.28.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
331
+ "gpt_neox.layers.28.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
332
+ "gpt_neox.layers.28.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
333
+ "gpt_neox.layers.28.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
334
+ "gpt_neox.layers.28.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
335
+ "gpt_neox.layers.28.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
336
+ "gpt_neox.layers.28.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
337
+ "gpt_neox.layers.28.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
338
+ "gpt_neox.layers.28.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
339
+ "gpt_neox.layers.28.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
340
+ "gpt_neox.layers.29.attention.bias": "pytorch_model-00002-of-00003.bin",
341
+ "gpt_neox.layers.29.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
342
+ "gpt_neox.layers.29.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
343
+ "gpt_neox.layers.29.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
344
+ "gpt_neox.layers.29.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
345
+ "gpt_neox.layers.29.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
346
+ "gpt_neox.layers.29.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
347
+ "gpt_neox.layers.29.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
348
+ "gpt_neox.layers.29.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
349
+ "gpt_neox.layers.29.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00003.bin",
350
+ "gpt_neox.layers.29.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00003.bin",
351
+ "gpt_neox.layers.29.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00003.bin",
352
+ "gpt_neox.layers.29.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00003.bin",
353
+ "gpt_neox.layers.29.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
354
+ "gpt_neox.layers.29.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
355
+ "gpt_neox.layers.3.attention.bias": "pytorch_model-00001-of-00003.bin",
356
+ "gpt_neox.layers.3.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
357
+ "gpt_neox.layers.3.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
358
+ "gpt_neox.layers.3.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
359
+ "gpt_neox.layers.3.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
360
+ "gpt_neox.layers.3.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
361
+ "gpt_neox.layers.3.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
362
+ "gpt_neox.layers.3.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
363
+ "gpt_neox.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
364
+ "gpt_neox.layers.3.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
365
+ "gpt_neox.layers.3.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
366
+ "gpt_neox.layers.3.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
367
+ "gpt_neox.layers.3.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
368
+ "gpt_neox.layers.3.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
369
+ "gpt_neox.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
370
+ "gpt_neox.layers.30.attention.bias": "pytorch_model-00002-of-00003.bin",
371
+ "gpt_neox.layers.30.attention.dense.bias": "pytorch_model-00002-of-00003.bin",
372
+ "gpt_neox.layers.30.attention.dense.weight": "pytorch_model-00002-of-00003.bin",
373
+ "gpt_neox.layers.30.attention.masked_bias": "pytorch_model-00002-of-00003.bin",
374
+ "gpt_neox.layers.30.attention.query_key_value.bias": "pytorch_model-00002-of-00003.bin",
375
+ "gpt_neox.layers.30.attention.query_key_value.weight": "pytorch_model-00002-of-00003.bin",
376
+ "gpt_neox.layers.30.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
377
+ "gpt_neox.layers.30.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
378
+ "gpt_neox.layers.30.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
379
+ "gpt_neox.layers.30.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00003.bin",
380
+ "gpt_neox.layers.30.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00003.bin",
381
+ "gpt_neox.layers.30.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00003.bin",
382
+ "gpt_neox.layers.30.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00003.bin",
383
+ "gpt_neox.layers.30.post_attention_layernorm.bias": "pytorch_model-00002-of-00003.bin",
384
+ "gpt_neox.layers.30.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
385
+ "gpt_neox.layers.31.attention.bias": "pytorch_model-00003-of-00003.bin",
386
+ "gpt_neox.layers.31.attention.dense.bias": "pytorch_model-00003-of-00003.bin",
387
+ "gpt_neox.layers.31.attention.dense.weight": "pytorch_model-00003-of-00003.bin",
388
+ "gpt_neox.layers.31.attention.masked_bias": "pytorch_model-00003-of-00003.bin",
389
+ "gpt_neox.layers.31.attention.query_key_value.bias": "pytorch_model-00003-of-00003.bin",
390
+ "gpt_neox.layers.31.attention.query_key_value.weight": "pytorch_model-00003-of-00003.bin",
391
+ "gpt_neox.layers.31.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
392
+ "gpt_neox.layers.31.input_layernorm.bias": "pytorch_model-00003-of-00003.bin",
393
+ "gpt_neox.layers.31.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
394
+ "gpt_neox.layers.31.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00003.bin",
395
+ "gpt_neox.layers.31.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00003.bin",
396
+ "gpt_neox.layers.31.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00003.bin",
397
+ "gpt_neox.layers.31.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00003.bin",
398
+ "gpt_neox.layers.31.post_attention_layernorm.bias": "pytorch_model-00003-of-00003.bin",
399
+ "gpt_neox.layers.31.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
400
+ "gpt_neox.layers.32.attention.bias": "pytorch_model-00003-of-00003.bin",
401
+ "gpt_neox.layers.32.attention.dense.bias": "pytorch_model-00003-of-00003.bin",
402
+ "gpt_neox.layers.32.attention.dense.weight": "pytorch_model-00003-of-00003.bin",
403
+ "gpt_neox.layers.32.attention.masked_bias": "pytorch_model-00003-of-00003.bin",
404
+ "gpt_neox.layers.32.attention.query_key_value.bias": "pytorch_model-00003-of-00003.bin",
405
+ "gpt_neox.layers.32.attention.query_key_value.weight": "pytorch_model-00003-of-00003.bin",
406
+ "gpt_neox.layers.32.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
407
+ "gpt_neox.layers.32.input_layernorm.bias": "pytorch_model-00003-of-00003.bin",
408
+ "gpt_neox.layers.32.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
409
+ "gpt_neox.layers.32.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00003.bin",
410
+ "gpt_neox.layers.32.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00003.bin",
411
+ "gpt_neox.layers.32.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00003.bin",
412
+ "gpt_neox.layers.32.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00003.bin",
413
+ "gpt_neox.layers.32.post_attention_layernorm.bias": "pytorch_model-00003-of-00003.bin",
414
+ "gpt_neox.layers.32.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
415
+ "gpt_neox.layers.33.attention.bias": "pytorch_model-00003-of-00003.bin",
416
+ "gpt_neox.layers.33.attention.dense.bias": "pytorch_model-00003-of-00003.bin",
417
+ "gpt_neox.layers.33.attention.dense.weight": "pytorch_model-00003-of-00003.bin",
418
+ "gpt_neox.layers.33.attention.masked_bias": "pytorch_model-00003-of-00003.bin",
419
+ "gpt_neox.layers.33.attention.query_key_value.bias": "pytorch_model-00003-of-00003.bin",
420
+ "gpt_neox.layers.33.attention.query_key_value.weight": "pytorch_model-00003-of-00003.bin",
421
+ "gpt_neox.layers.33.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
422
+ "gpt_neox.layers.33.input_layernorm.bias": "pytorch_model-00003-of-00003.bin",
423
+ "gpt_neox.layers.33.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
424
+ "gpt_neox.layers.33.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00003.bin",
425
+ "gpt_neox.layers.33.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00003.bin",
426
+ "gpt_neox.layers.33.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00003.bin",
427
+ "gpt_neox.layers.33.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00003.bin",
428
+ "gpt_neox.layers.33.post_attention_layernorm.bias": "pytorch_model-00003-of-00003.bin",
429
+ "gpt_neox.layers.33.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
430
+ "gpt_neox.layers.34.attention.bias": "pytorch_model-00003-of-00003.bin",
431
+ "gpt_neox.layers.34.attention.dense.bias": "pytorch_model-00003-of-00003.bin",
432
+ "gpt_neox.layers.34.attention.dense.weight": "pytorch_model-00003-of-00003.bin",
433
+ "gpt_neox.layers.34.attention.masked_bias": "pytorch_model-00003-of-00003.bin",
434
+ "gpt_neox.layers.34.attention.query_key_value.bias": "pytorch_model-00003-of-00003.bin",
435
+ "gpt_neox.layers.34.attention.query_key_value.weight": "pytorch_model-00003-of-00003.bin",
436
+ "gpt_neox.layers.34.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
437
+ "gpt_neox.layers.34.input_layernorm.bias": "pytorch_model-00003-of-00003.bin",
438
+ "gpt_neox.layers.34.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
439
+ "gpt_neox.layers.34.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00003.bin",
440
+ "gpt_neox.layers.34.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00003.bin",
441
+ "gpt_neox.layers.34.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00003.bin",
442
+ "gpt_neox.layers.34.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00003.bin",
443
+ "gpt_neox.layers.34.post_attention_layernorm.bias": "pytorch_model-00003-of-00003.bin",
444
+ "gpt_neox.layers.34.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
445
+ "gpt_neox.layers.35.attention.bias": "pytorch_model-00003-of-00003.bin",
446
+ "gpt_neox.layers.35.attention.dense.bias": "pytorch_model-00003-of-00003.bin",
447
+ "gpt_neox.layers.35.attention.dense.weight": "pytorch_model-00003-of-00003.bin",
448
+ "gpt_neox.layers.35.attention.masked_bias": "pytorch_model-00003-of-00003.bin",
449
+ "gpt_neox.layers.35.attention.query_key_value.bias": "pytorch_model-00003-of-00003.bin",
450
+ "gpt_neox.layers.35.attention.query_key_value.weight": "pytorch_model-00003-of-00003.bin",
451
+ "gpt_neox.layers.35.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
452
+ "gpt_neox.layers.35.input_layernorm.bias": "pytorch_model-00003-of-00003.bin",
453
+ "gpt_neox.layers.35.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
454
+ "gpt_neox.layers.35.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00003.bin",
455
+ "gpt_neox.layers.35.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00003.bin",
456
+ "gpt_neox.layers.35.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00003.bin",
457
+ "gpt_neox.layers.35.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00003.bin",
458
+ "gpt_neox.layers.35.post_attention_layernorm.bias": "pytorch_model-00003-of-00003.bin",
459
+ "gpt_neox.layers.35.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
460
+ "gpt_neox.layers.4.attention.bias": "pytorch_model-00001-of-00003.bin",
461
+ "gpt_neox.layers.4.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
462
+ "gpt_neox.layers.4.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
463
+ "gpt_neox.layers.4.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
464
+ "gpt_neox.layers.4.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
465
+ "gpt_neox.layers.4.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
466
+ "gpt_neox.layers.4.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
467
+ "gpt_neox.layers.4.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
468
+ "gpt_neox.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
469
+ "gpt_neox.layers.4.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
470
+ "gpt_neox.layers.4.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
471
+ "gpt_neox.layers.4.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
472
+ "gpt_neox.layers.4.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
473
+ "gpt_neox.layers.4.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
474
+ "gpt_neox.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
475
+ "gpt_neox.layers.5.attention.bias": "pytorch_model-00001-of-00003.bin",
476
+ "gpt_neox.layers.5.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
477
+ "gpt_neox.layers.5.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
478
+ "gpt_neox.layers.5.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
479
+ "gpt_neox.layers.5.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
480
+ "gpt_neox.layers.5.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
481
+ "gpt_neox.layers.5.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
482
+ "gpt_neox.layers.5.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
483
+ "gpt_neox.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
484
+ "gpt_neox.layers.5.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
485
+ "gpt_neox.layers.5.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
486
+ "gpt_neox.layers.5.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
487
+ "gpt_neox.layers.5.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
488
+ "gpt_neox.layers.5.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
489
+ "gpt_neox.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
490
+ "gpt_neox.layers.6.attention.bias": "pytorch_model-00001-of-00003.bin",
491
+ "gpt_neox.layers.6.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
492
+ "gpt_neox.layers.6.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
493
+ "gpt_neox.layers.6.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
494
+ "gpt_neox.layers.6.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
495
+ "gpt_neox.layers.6.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
496
+ "gpt_neox.layers.6.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
497
+ "gpt_neox.layers.6.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
498
+ "gpt_neox.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
499
+ "gpt_neox.layers.6.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
500
+ "gpt_neox.layers.6.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
501
+ "gpt_neox.layers.6.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
502
+ "gpt_neox.layers.6.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
503
+ "gpt_neox.layers.6.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
504
+ "gpt_neox.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
505
+ "gpt_neox.layers.7.attention.bias": "pytorch_model-00001-of-00003.bin",
506
+ "gpt_neox.layers.7.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
507
+ "gpt_neox.layers.7.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
508
+ "gpt_neox.layers.7.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
509
+ "gpt_neox.layers.7.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
510
+ "gpt_neox.layers.7.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
511
+ "gpt_neox.layers.7.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
512
+ "gpt_neox.layers.7.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
513
+ "gpt_neox.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
514
+ "gpt_neox.layers.7.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
515
+ "gpt_neox.layers.7.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
516
+ "gpt_neox.layers.7.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
517
+ "gpt_neox.layers.7.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
518
+ "gpt_neox.layers.7.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
519
+ "gpt_neox.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
520
+ "gpt_neox.layers.8.attention.bias": "pytorch_model-00001-of-00003.bin",
521
+ "gpt_neox.layers.8.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
522
+ "gpt_neox.layers.8.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
523
+ "gpt_neox.layers.8.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
524
+ "gpt_neox.layers.8.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
525
+ "gpt_neox.layers.8.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
526
+ "gpt_neox.layers.8.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
527
+ "gpt_neox.layers.8.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
528
+ "gpt_neox.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
529
+ "gpt_neox.layers.8.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
530
+ "gpt_neox.layers.8.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
531
+ "gpt_neox.layers.8.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
532
+ "gpt_neox.layers.8.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
533
+ "gpt_neox.layers.8.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
534
+ "gpt_neox.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
535
+ "gpt_neox.layers.9.attention.bias": "pytorch_model-00001-of-00003.bin",
536
+ "gpt_neox.layers.9.attention.dense.bias": "pytorch_model-00001-of-00003.bin",
537
+ "gpt_neox.layers.9.attention.dense.weight": "pytorch_model-00001-of-00003.bin",
538
+ "gpt_neox.layers.9.attention.masked_bias": "pytorch_model-00001-of-00003.bin",
539
+ "gpt_neox.layers.9.attention.query_key_value.bias": "pytorch_model-00001-of-00003.bin",
540
+ "gpt_neox.layers.9.attention.query_key_value.weight": "pytorch_model-00001-of-00003.bin",
541
+ "gpt_neox.layers.9.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
542
+ "gpt_neox.layers.9.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
543
+ "gpt_neox.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
544
+ "gpt_neox.layers.9.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00003.bin",
545
+ "gpt_neox.layers.9.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00003.bin",
546
+ "gpt_neox.layers.9.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00003.bin",
547
+ "gpt_neox.layers.9.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00003.bin",
548
+ "gpt_neox.layers.9.post_attention_layernorm.bias": "pytorch_model-00001-of-00003.bin",
549
+ "gpt_neox.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin"
550
+ }
551
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "</prefix>",
4
+ "<human>",
5
+ "<bot>",
6
+ "<prefix>"
7
+ ],
8
+ "bos_token": "<|endoftext|>",
9
+ "eos_token": "<|endoftext|>",
10
+ "pad_token": "<|padding|>",
11
+ "sep_token": "<|endoftext|>",
12
+ "unk_token": "<|endoftext|>"
13
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": "<|endoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "model_max_length": 1000000000000000019884624838656,
6
+ "name_or_path": "EleutherAI/pythia-12b-deduped",
7
+ "special_tokens_map_file": "/fsx/home-hailey/.cache/huggingface/hub/models--EleutherAI--gpt-neox-20b/snapshots/3523781c8df75f7741687a4284f6f70e1afa12f4/special_tokens_map.json",
8
+ "tokenizer_class": "GPTNeoXTokenizer",
9
+ "unk_token": "<|endoftext|>"
10
+ }
trainer_state.json ADDED
@@ -0,0 +1,1804 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.029488434635935788,
5
+ "global_step": 1000,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.0,
12
+ "learning_rate": 1.4084967333570947e-06,
13
+ "loss": 2.0409,
14
+ "step": 10
15
+ },
16
+ {
17
+ "epoch": 0.0,
18
+ "learning_rate": 2.0507482022971233e-06,
19
+ "loss": 1.8182,
20
+ "step": 20
21
+ },
22
+ {
23
+ "epoch": 0.0,
24
+ "learning_rate": 2.385606273598312e-06,
25
+ "loss": 1.6715,
26
+ "step": 30
27
+ },
28
+ {
29
+ "epoch": 0.0,
30
+ "learning_rate": 2.6136695401116585e-06,
31
+ "loss": 1.7864,
32
+ "step": 40
33
+ },
34
+ {
35
+ "epoch": 0.0,
36
+ "learning_rate": 2.7868297632261957e-06,
37
+ "loss": 1.6323,
38
+ "step": 50
39
+ },
40
+ {
41
+ "epoch": 0.0,
42
+ "learning_rate": 2.926458092787486e-06,
43
+ "loss": 1.6948,
44
+ "step": 60
45
+ },
46
+ {
47
+ "epoch": 0.0,
48
+ "learning_rate": 3.0434580045013773e-06,
49
+ "loss": 1.6492,
50
+ "step": 70
51
+ },
52
+ {
53
+ "epoch": 0.0,
54
+ "learning_rate": 3.1441512086208035e-06,
55
+ "loss": 1.6454,
56
+ "step": 80
57
+ },
58
+ {
59
+ "epoch": 0.0,
60
+ "learning_rate": 3.232532087697698e-06,
61
+ "loss": 1.6685,
62
+ "step": 90
63
+ },
64
+ {
65
+ "epoch": 0.0,
66
+ "learning_rate": 3.3112862237770753e-06,
67
+ "loss": 1.5988,
68
+ "step": 100
69
+ },
70
+ {
71
+ "epoch": 0.0,
72
+ "learning_rate": 3.3823062961420163e-06,
73
+ "loss": 1.5397,
74
+ "step": 110
75
+ },
76
+ {
77
+ "epoch": 0.0,
78
+ "learning_rate": 3.446976436243603e-06,
79
+ "loss": 1.6389,
80
+ "step": 120
81
+ },
82
+ {
83
+ "epoch": 0.0,
84
+ "learning_rate": 3.506339534926595e-06,
85
+ "loss": 1.5864,
86
+ "step": 130
87
+ },
88
+ {
89
+ "epoch": 0.0,
90
+ "learning_rate": 3.5612009452606784e-06,
91
+ "loss": 1.6896,
92
+ "step": 140
93
+ },
94
+ {
95
+ "epoch": 0.0,
96
+ "learning_rate": 3.612195557913627e-06,
97
+ "loss": 1.6217,
98
+ "step": 150
99
+ },
100
+ {
101
+ "epoch": 0.0,
102
+ "learning_rate": 3.65983275401539e-06,
103
+ "loss": 1.6265,
104
+ "step": 160
105
+ },
106
+ {
107
+ "epoch": 0.01,
108
+ "learning_rate": 3.7045274519126395e-06,
109
+ "loss": 1.6128,
110
+ "step": 170
111
+ },
112
+ {
113
+ "epoch": 0.01,
114
+ "learning_rate": 3.7466221106030114e-06,
115
+ "loss": 1.5282,
116
+ "step": 180
117
+ },
118
+ {
119
+ "epoch": 0.01,
120
+ "learning_rate": 3.786402677560832e-06,
121
+ "loss": 1.6623,
122
+ "step": 190
123
+ },
124
+ {
125
+ "epoch": 0.01,
126
+ "learning_rate": 3.824110376935989e-06,
127
+ "loss": 1.5958,
128
+ "step": 200
129
+ },
130
+ {
131
+ "epoch": 0.01,
132
+ "learning_rate": 3.8599505757615295e-06,
133
+ "loss": 1.6162,
134
+ "step": 210
135
+ },
136
+ {
137
+ "epoch": 0.01,
138
+ "learning_rate": 3.894099556414216e-06,
139
+ "loss": 1.5936,
140
+ "step": 220
141
+ },
142
+ {
143
+ "epoch": 0.01,
144
+ "learning_rate": 3.9267097619885385e-06,
145
+ "loss": 1.5658,
146
+ "step": 230
147
+ },
148
+ {
149
+ "epoch": 0.01,
150
+ "learning_rate": 3.95791391001684e-06,
151
+ "loss": 1.627,
152
+ "step": 240
153
+ },
154
+ {
155
+ "epoch": 0.01,
156
+ "learning_rate": 3.987828255432777e-06,
157
+ "loss": 1.6155,
158
+ "step": 250
159
+ },
160
+ {
161
+ "epoch": 0.01,
162
+ "eval_gsm8k_hard_accuracy": 0.8835755873132202,
163
+ "eval_gsm8k_hard_loss": 0.51513671875,
164
+ "eval_gsm8k_hard_runtime": 6.8667,
165
+ "eval_gsm8k_hard_samples_per_second": 38.446,
166
+ "eval_gsm8k_hard_steps_per_second": 0.437,
167
+ "step": 250
168
+ },
169
+ {
170
+ "epoch": 0.01,
171
+ "eval_webgpt_accuracy": 0.49985688262224126,
172
+ "eval_webgpt_loss": 2.197265625,
173
+ "eval_webgpt_runtime": 38.8229,
174
+ "eval_webgpt_samples_per_second": 100.868,
175
+ "eval_webgpt_steps_per_second": 1.056,
176
+ "step": 250
177
+ },
178
+ {
179
+ "epoch": 0.01,
180
+ "eval_squad_v2_accuracy": 0.8732225651432517,
181
+ "eval_squad_v2_loss": 0.394775390625,
182
+ "eval_squad_v2_runtime": 212.1787,
183
+ "eval_squad_v2_samples_per_second": 122.84,
184
+ "eval_squad_v2_steps_per_second": 1.282,
185
+ "step": 250
186
+ },
187
+ {
188
+ "epoch": 0.01,
189
+ "eval_adversarial_qa_accuracy": 0.7885436255634161,
190
+ "eval_adversarial_qa_loss": 0.84423828125,
191
+ "eval_adversarial_qa_runtime": 53.4145,
192
+ "eval_adversarial_qa_samples_per_second": 112.329,
193
+ "eval_adversarial_qa_steps_per_second": 1.179,
194
+ "step": 250
195
+ },
196
+ {
197
+ "epoch": 0.01,
198
+ "eval_private_tuning_accuracy": 0.6697048468296535,
199
+ "eval_private_tuning_loss": 1.234375,
200
+ "eval_private_tuning_runtime": 147.3821,
201
+ "eval_private_tuning_samples_per_second": 143.695,
202
+ "eval_private_tuning_steps_per_second": 1.5,
203
+ "step": 250
204
+ },
205
+ {
206
+ "epoch": 0.01,
207
+ "eval_oa_translated_accuracy": 0.6888436472902636,
208
+ "eval_oa_translated_loss": 1.271484375,
209
+ "eval_oa_translated_runtime": 1288.641,
210
+ "eval_oa_translated_samples_per_second": 91.051,
211
+ "eval_oa_translated_steps_per_second": 0.949,
212
+ "step": 250
213
+ },
214
+ {
215
+ "epoch": 0.01,
216
+ "eval_prosocial_dialogue_accuracy": 0.5277240036359349,
217
+ "eval_prosocial_dialogue_loss": 1.830078125,
218
+ "eval_prosocial_dialogue_runtime": 61.2751,
219
+ "eval_prosocial_dialogue_samples_per_second": 440.358,
220
+ "eval_prosocial_dialogue_steps_per_second": 4.602,
221
+ "step": 250
222
+ },
223
+ {
224
+ "epoch": 0.01,
225
+ "eval_math_qa_accuracy": 0.5650762200656649,
226
+ "eval_math_qa_loss": 1.912109375,
227
+ "eval_math_qa_runtime": 43.4013,
228
+ "eval_math_qa_samples_per_second": 137.507,
229
+ "eval_math_qa_steps_per_second": 1.452,
230
+ "step": 250
231
+ },
232
+ {
233
+ "epoch": 0.01,
234
+ "eval_wikihow_accuracy": 0.6096311191235613,
235
+ "eval_wikihow_loss": 1.8701171875,
236
+ "eval_wikihow_runtime": 16.5775,
237
+ "eval_wikihow_samples_per_second": 138.32,
238
+ "eval_wikihow_steps_per_second": 1.448,
239
+ "step": 250
240
+ },
241
+ {
242
+ "epoch": 0.01,
243
+ "eval_joke_accuracy": 0.49194465504169826,
244
+ "eval_joke_loss": 2.216796875,
245
+ "eval_joke_runtime": 2.4324,
246
+ "eval_joke_samples_per_second": 31.245,
247
+ "eval_joke_steps_per_second": 0.411,
248
+ "step": 250
249
+ },
250
+ {
251
+ "epoch": 0.01,
252
+ "eval_gsm8k_accuracy": 0.7494717398301172,
253
+ "eval_gsm8k_loss": 0.9765625,
254
+ "eval_gsm8k_runtime": 10.7538,
255
+ "eval_gsm8k_samples_per_second": 139.021,
256
+ "eval_gsm8k_steps_per_second": 1.488,
257
+ "step": 250
258
+ },
259
+ {
260
+ "epoch": 0.01,
261
+ "eval_ted_trans_en-hi_accuracy": 0.6902974158946855,
262
+ "eval_ted_trans_en-hi_loss": 1.1455078125,
263
+ "eval_ted_trans_en-hi_runtime": 4.0656,
264
+ "eval_ted_trans_en-hi_samples_per_second": 25.335,
265
+ "eval_ted_trans_en-hi_steps_per_second": 0.492,
266
+ "step": 250
267
+ },
268
+ {
269
+ "epoch": 0.01,
270
+ "eval_ted_trans_de-ja_accuracy": 0.6504194889162561,
271
+ "eval_ted_trans_de-ja_loss": 1.52734375,
272
+ "eval_ted_trans_de-ja_runtime": 8.8337,
273
+ "eval_ted_trans_de-ja_samples_per_second": 81.279,
274
+ "eval_ted_trans_de-ja_steps_per_second": 0.906,
275
+ "step": 250
276
+ },
277
+ {
278
+ "epoch": 0.01,
279
+ "eval_ted_trans_nl-en_accuracy": 0.7506186489759386,
280
+ "eval_ted_trans_nl-en_loss": 1.091796875,
281
+ "eval_ted_trans_nl-en_runtime": 8.871,
282
+ "eval_ted_trans_nl-en_samples_per_second": 86.913,
283
+ "eval_ted_trans_nl-en_steps_per_second": 1.015,
284
+ "step": 250
285
+ },
286
+ {
287
+ "epoch": 0.01,
288
+ "eval_ted_trans_en-ja_accuracy": 0.6557647009776333,
289
+ "eval_ted_trans_en-ja_loss": 1.427734375,
290
+ "eval_ted_trans_en-ja_runtime": 9.6821,
291
+ "eval_ted_trans_en-ja_samples_per_second": 82.73,
292
+ "eval_ted_trans_en-ja_steps_per_second": 0.93,
293
+ "step": 250
294
+ },
295
+ {
296
+ "epoch": 0.01,
297
+ "eval_ted_trans_en-es_accuracy": 0.7831022379328372,
298
+ "eval_ted_trans_en-es_loss": 0.89599609375,
299
+ "eval_ted_trans_en-es_runtime": 7.8367,
300
+ "eval_ted_trans_en-es_samples_per_second": 105.401,
301
+ "eval_ted_trans_en-es_steps_per_second": 1.148,
302
+ "step": 250
303
+ },
304
+ {
305
+ "epoch": 0.01,
306
+ "eval_ted_trans_en-ms_accuracy": 0.689470871191876,
307
+ "eval_ted_trans_en-ms_loss": 1.4052734375,
308
+ "eval_ted_trans_en-ms_runtime": 1.3714,
309
+ "eval_ted_trans_en-ms_samples_per_second": 30.625,
310
+ "eval_ted_trans_en-ms_steps_per_second": 0.729,
311
+ "step": 250
312
+ },
313
+ {
314
+ "epoch": 0.01,
315
+ "eval_xsum_accuracy": 0.6193833980292625,
316
+ "eval_xsum_loss": 1.4599609375,
317
+ "eval_xsum_runtime": 434.5368,
318
+ "eval_xsum_samples_per_second": 93.914,
319
+ "eval_xsum_steps_per_second": 0.98,
320
+ "step": 250
321
+ },
322
+ {
323
+ "epoch": 0.01,
324
+ "eval_cnn_dailymail_accuracy": 0.6712703040399833,
325
+ "eval_cnn_dailymail_loss": NaN,
326
+ "eval_cnn_dailymail_runtime": 624.2796,
327
+ "eval_cnn_dailymail_samples_per_second": 91.983,
328
+ "eval_cnn_dailymail_steps_per_second": 0.96,
329
+ "step": 250
330
+ },
331
+ {
332
+ "epoch": 0.01,
333
+ "eval_multi_news_accuracy": 0.5461545789406833,
334
+ "eval_multi_news_loss": NaN,
335
+ "eval_multi_news_runtime": 102.6315,
336
+ "eval_multi_news_samples_per_second": 87.644,
337
+ "eval_multi_news_steps_per_second": 0.916,
338
+ "step": 250
339
+ },
340
+ {
341
+ "epoch": 0.01,
342
+ "eval_tldr_news_accuracy": 0.5329163923633969,
343
+ "eval_tldr_news_loss": 2.1640625,
344
+ "eval_tldr_news_runtime": 7.304,
345
+ "eval_tldr_news_samples_per_second": 195.509,
346
+ "eval_tldr_news_steps_per_second": 2.054,
347
+ "step": 250
348
+ },
349
+ {
350
+ "epoch": 0.01,
351
+ "eval_scitldr_accuracy": 0.5056726094003241,
352
+ "eval_scitldr_loss": NaN,
353
+ "eval_scitldr_runtime": 6.0172,
354
+ "eval_scitldr_samples_per_second": 66.309,
355
+ "eval_scitldr_steps_per_second": 0.831,
356
+ "step": 250
357
+ },
358
+ {
359
+ "epoch": 0.01,
360
+ "eval_samsum_accuracy": 0.6255323175925049,
361
+ "eval_samsum_loss": 1.390625,
362
+ "eval_samsum_runtime": 31.2731,
363
+ "eval_samsum_samples_per_second": 94.234,
364
+ "eval_samsum_steps_per_second": 0.991,
365
+ "step": 250
366
+ },
367
+ {
368
+ "epoch": 0.01,
369
+ "eval_debate_sum_accuracy": 0.934249098160658,
370
+ "eval_debate_sum_loss": 0.363525390625,
371
+ "eval_debate_sum_runtime": 539.9242,
372
+ "eval_debate_sum_samples_per_second": 89.113,
373
+ "eval_debate_sum_steps_per_second": 0.93,
374
+ "step": 250
375
+ },
376
+ {
377
+ "epoch": 0.01,
378
+ "eval_billsum_accuracy": 0.6761362636279469,
379
+ "eval_billsum_loss": 1.3681640625,
380
+ "eval_billsum_runtime": 47.9835,
381
+ "eval_billsum_samples_per_second": 78.985,
382
+ "eval_billsum_steps_per_second": 0.834,
383
+ "step": 250
384
+ },
385
+ {
386
+ "epoch": 0.01,
387
+ "eval_wmt2019_zh-en_accuracy": 0.6633805205208717,
388
+ "eval_wmt2019_zh-en_loss": 1.474609375,
389
+ "eval_wmt2019_zh-en_runtime": 27.1758,
390
+ "eval_wmt2019_zh-en_samples_per_second": 146.491,
391
+ "eval_wmt2019_zh-en_steps_per_second": 1.545,
392
+ "step": 250
393
+ },
394
+ {
395
+ "epoch": 0.01,
396
+ "eval_wmt2019_ru-en_accuracy": 0.7568385011868931,
397
+ "eval_wmt2019_ru-en_loss": 0.9365234375,
398
+ "eval_wmt2019_ru-en_runtime": 21.7646,
399
+ "eval_wmt2019_ru-en_samples_per_second": 137.839,
400
+ "eval_wmt2019_ru-en_steps_per_second": 1.47,
401
+ "step": 250
402
+ },
403
+ {
404
+ "epoch": 0.01,
405
+ "eval_wmt2019_de-en_accuracy": 0.7579152898768399,
406
+ "eval_wmt2019_de-en_loss": 0.94921875,
407
+ "eval_wmt2019_de-en_runtime": 15.095,
408
+ "eval_wmt2019_de-en_samples_per_second": 198.609,
409
+ "eval_wmt2019_de-en_steps_per_second": 2.12,
410
+ "step": 250
411
+ },
412
+ {
413
+ "epoch": 0.01,
414
+ "eval_wmt2019_fr-de_accuracy": 0.7458755561047948,
415
+ "eval_wmt2019_fr-de_loss": 1.0107421875,
416
+ "eval_wmt2019_fr-de_runtime": 10.8089,
417
+ "eval_wmt2019_fr-de_samples_per_second": 139.885,
418
+ "eval_wmt2019_fr-de_steps_per_second": 1.48,
419
+ "step": 250
420
+ },
421
+ {
422
+ "epoch": 0.01,
423
+ "eval_essay_instruction_accuracy": 0.5980087566061517,
424
+ "eval_essay_instruction_loss": 1.939453125,
425
+ "eval_essay_instruction_runtime": 8.5102,
426
+ "eval_essay_instruction_samples_per_second": 48.53,
427
+ "eval_essay_instruction_steps_per_second": 0.588,
428
+ "step": 250
429
+ },
430
+ {
431
+ "epoch": 0.01,
432
+ "eval_reddit_eli5_accuracy": 0.4587571011238715,
433
+ "eval_reddit_eli5_loss": 2.43359375,
434
+ "eval_reddit_eli5_runtime": 592.806,
435
+ "eval_reddit_eli5_samples_per_second": 91.981,
436
+ "eval_reddit_eli5_steps_per_second": 0.958,
437
+ "step": 250
438
+ },
439
+ {
440
+ "epoch": 0.01,
441
+ "eval_reddit_askh_accuracy": 0.46236593037589085,
442
+ "eval_reddit_askh_loss": 2.53125,
443
+ "eval_reddit_askh_runtime": 245.8916,
444
+ "eval_reddit_askh_samples_per_second": 80.137,
445
+ "eval_reddit_askh_steps_per_second": 0.838,
446
+ "step": 250
447
+ },
448
+ {
449
+ "epoch": 0.01,
450
+ "eval_reddit_asks_accuracy": 0.4693832359074744,
451
+ "eval_reddit_asks_loss": 2.390625,
452
+ "eval_reddit_asks_runtime": 307.0019,
453
+ "eval_reddit_asks_samples_per_second": 85.85,
454
+ "eval_reddit_asks_steps_per_second": 0.896,
455
+ "step": 250
456
+ },
457
+ {
458
+ "epoch": 0.01,
459
+ "learning_rate": 4.016555205552159e-06,
460
+ "loss": 1.6024,
461
+ "step": 260
462
+ },
463
+ {
464
+ "epoch": 0.01,
465
+ "learning_rate": 4.044185435607626e-06,
466
+ "loss": 1.6344,
467
+ "step": 270
468
+ },
469
+ {
470
+ "epoch": 0.01,
471
+ "learning_rate": 4.070799615107415e-06,
472
+ "loss": 1.5251,
473
+ "step": 280
474
+ },
475
+ {
476
+ "epoch": 0.01,
477
+ "learning_rate": 4.096469827889988e-06,
478
+ "loss": 1.5818,
479
+ "step": 290
480
+ },
481
+ {
482
+ "epoch": 0.01,
483
+ "learning_rate": 4.121260748862021e-06,
484
+ "loss": 1.6331,
485
+ "step": 300
486
+ },
487
+ {
488
+ "epoch": 0.01,
489
+ "learning_rate": 4.145230625795312e-06,
490
+ "loss": 1.6272,
491
+ "step": 310
492
+ },
493
+ {
494
+ "epoch": 0.01,
495
+ "learning_rate": 4.1684321036962525e-06,
496
+ "loss": 1.5872,
497
+ "step": 320
498
+ },
499
+ {
500
+ "epoch": 0.01,
501
+ "learning_rate": 4.190912921100477e-06,
502
+ "loss": 1.6504,
503
+ "step": 330
504
+ },
505
+ {
506
+ "epoch": 0.01,
507
+ "learning_rate": 4.212716501452232e-06,
508
+ "loss": 1.566,
509
+ "step": 340
510
+ },
511
+ {
512
+ "epoch": 0.01,
513
+ "learning_rate": 4.233882457984791e-06,
514
+ "loss": 1.6106,
515
+ "step": 350
516
+ },
517
+ {
518
+ "epoch": 0.01,
519
+ "learning_rate": 4.2544470268536555e-06,
520
+ "loss": 1.616,
521
+ "step": 360
522
+ },
523
+ {
524
+ "epoch": 0.01,
525
+ "learning_rate": 4.27444344042015e-06,
526
+ "loss": 1.6374,
527
+ "step": 370
528
+ },
529
+ {
530
+ "epoch": 0.01,
531
+ "learning_rate": 4.293902250342989e-06,
532
+ "loss": 1.5941,
533
+ "step": 380
534
+ },
535
+ {
536
+ "epoch": 0.01,
537
+ "learning_rate": 4.312851608364853e-06,
538
+ "loss": 1.6115,
539
+ "step": 390
540
+ },
541
+ {
542
+ "epoch": 0.01,
543
+ "learning_rate": 4.3313175112718595e-06,
544
+ "loss": 1.5531,
545
+ "step": 400
546
+ },
547
+ {
548
+ "epoch": 0.01,
549
+ "learning_rate": 4.3493240153753665e-06,
550
+ "loss": 1.5554,
551
+ "step": 410
552
+ },
553
+ {
554
+ "epoch": 0.01,
555
+ "learning_rate": 4.366893424956263e-06,
556
+ "loss": 1.5233,
557
+ "step": 420
558
+ },
559
+ {
560
+ "epoch": 0.01,
561
+ "learning_rate": 4.38404645837504e-06,
562
+ "loss": 1.5579,
563
+ "step": 430
564
+ },
565
+ {
566
+ "epoch": 0.01,
567
+ "learning_rate": 4.400802394950703e-06,
568
+ "loss": 1.6028,
569
+ "step": 440
570
+ },
571
+ {
572
+ "epoch": 0.01,
573
+ "learning_rate": 4.4171792052198945e-06,
574
+ "loss": 1.5239,
575
+ "step": 450
576
+ },
577
+ {
578
+ "epoch": 0.01,
579
+ "learning_rate": 4.433193666783084e-06,
580
+ "loss": 1.6149,
581
+ "step": 460
582
+ },
583
+ {
584
+ "epoch": 0.01,
585
+ "learning_rate": 4.448861467610187e-06,
586
+ "loss": 1.6114,
587
+ "step": 470
588
+ },
589
+ {
590
+ "epoch": 0.01,
591
+ "learning_rate": 4.4641972984001906e-06,
592
+ "loss": 1.6682,
593
+ "step": 480
594
+ },
595
+ {
596
+ "epoch": 0.01,
597
+ "learning_rate": 4.479214935357724e-06,
598
+ "loss": 1.5707,
599
+ "step": 490
600
+ },
601
+ {
602
+ "epoch": 0.01,
603
+ "learning_rate": 4.493927314555554e-06,
604
+ "loss": 1.5827,
605
+ "step": 500
606
+ },
607
+ {
608
+ "epoch": 0.01,
609
+ "eval_gsm8k_hard_accuracy": 0.9183054435894304,
610
+ "eval_gsm8k_hard_loss": 0.3740234375,
611
+ "eval_gsm8k_hard_runtime": 3.8504,
612
+ "eval_gsm8k_hard_samples_per_second": 68.565,
613
+ "eval_gsm8k_hard_steps_per_second": 0.779,
614
+ "step": 500
615
+ },
616
+ {
617
+ "epoch": 0.01,
618
+ "eval_webgpt_accuracy": 0.502614994399395,
619
+ "eval_webgpt_loss": 2.185546875,
620
+ "eval_webgpt_runtime": 37.4417,
621
+ "eval_webgpt_samples_per_second": 104.589,
622
+ "eval_webgpt_steps_per_second": 1.095,
623
+ "step": 500
624
+ },
625
+ {
626
+ "epoch": 0.01,
627
+ "eval_squad_v2_accuracy": 0.8982722417170479,
628
+ "eval_squad_v2_loss": 0.33447265625,
629
+ "eval_squad_v2_runtime": 214.9352,
630
+ "eval_squad_v2_samples_per_second": 121.264,
631
+ "eval_squad_v2_steps_per_second": 1.265,
632
+ "step": 500
633
+ },
634
+ {
635
+ "epoch": 0.01,
636
+ "eval_adversarial_qa_accuracy": 0.8046625473866452,
637
+ "eval_adversarial_qa_loss": 0.8486328125,
638
+ "eval_adversarial_qa_runtime": 51.9881,
639
+ "eval_adversarial_qa_samples_per_second": 115.411,
640
+ "eval_adversarial_qa_steps_per_second": 1.212,
641
+ "step": 500
642
+ },
643
+ {
644
+ "epoch": 0.01,
645
+ "eval_private_tuning_accuracy": 0.6754279825666092,
646
+ "eval_private_tuning_loss": 1.20703125,
647
+ "eval_private_tuning_runtime": 143.688,
648
+ "eval_private_tuning_samples_per_second": 147.389,
649
+ "eval_private_tuning_steps_per_second": 1.538,
650
+ "step": 500
651
+ },
652
+ {
653
+ "epoch": 0.01,
654
+ "eval_oa_translated_accuracy": 0.6956755454438557,
655
+ "eval_oa_translated_loss": 1.2421875,
656
+ "eval_oa_translated_runtime": 1298.0566,
657
+ "eval_oa_translated_samples_per_second": 90.391,
658
+ "eval_oa_translated_steps_per_second": 0.942,
659
+ "step": 500
660
+ },
661
+ {
662
+ "epoch": 0.01,
663
+ "eval_prosocial_dialogue_accuracy": 0.5306487253309804,
664
+ "eval_prosocial_dialogue_loss": 1.783203125,
665
+ "eval_prosocial_dialogue_runtime": 62.6995,
666
+ "eval_prosocial_dialogue_samples_per_second": 430.355,
667
+ "eval_prosocial_dialogue_steps_per_second": 4.498,
668
+ "step": 500
669
+ },
670
+ {
671
+ "epoch": 0.01,
672
+ "eval_math_qa_accuracy": 0.573035368807606,
673
+ "eval_math_qa_loss": 1.849609375,
674
+ "eval_math_qa_runtime": 42.0578,
675
+ "eval_math_qa_samples_per_second": 141.9,
676
+ "eval_math_qa_steps_per_second": 1.498,
677
+ "step": 500
678
+ },
679
+ {
680
+ "epoch": 0.01,
681
+ "eval_wikihow_accuracy": 0.6166412425461101,
682
+ "eval_wikihow_loss": 1.8369140625,
683
+ "eval_wikihow_runtime": 17.5874,
684
+ "eval_wikihow_samples_per_second": 130.377,
685
+ "eval_wikihow_steps_per_second": 1.365,
686
+ "step": 500
687
+ },
688
+ {
689
+ "epoch": 0.01,
690
+ "eval_joke_accuracy": 0.500284306292646,
691
+ "eval_joke_loss": 2.1875,
692
+ "eval_joke_runtime": 1.5291,
693
+ "eval_joke_samples_per_second": 49.704,
694
+ "eval_joke_steps_per_second": 0.654,
695
+ "step": 500
696
+ },
697
+ {
698
+ "epoch": 0.01,
699
+ "eval_gsm8k_accuracy": 0.7605687018093785,
700
+ "eval_gsm8k_loss": 0.91357421875,
701
+ "eval_gsm8k_runtime": 11.3759,
702
+ "eval_gsm8k_samples_per_second": 131.418,
703
+ "eval_gsm8k_steps_per_second": 1.406,
704
+ "step": 500
705
+ },
706
+ {
707
+ "epoch": 0.01,
708
+ "eval_ted_trans_en-hi_accuracy": 0.6839278864595321,
709
+ "eval_ted_trans_en-hi_loss": 1.142578125,
710
+ "eval_ted_trans_en-hi_runtime": 2.7736,
711
+ "eval_ted_trans_en-hi_samples_per_second": 37.135,
712
+ "eval_ted_trans_en-hi_steps_per_second": 0.721,
713
+ "step": 500
714
+ },
715
+ {
716
+ "epoch": 0.01,
717
+ "eval_ted_trans_de-ja_accuracy": 0.6501228312605558,
718
+ "eval_ted_trans_de-ja_loss": 1.5048828125,
719
+ "eval_ted_trans_de-ja_runtime": 8.2515,
720
+ "eval_ted_trans_de-ja_samples_per_second": 87.014,
721
+ "eval_ted_trans_de-ja_steps_per_second": 0.97,
722
+ "step": 500
723
+ },
724
+ {
725
+ "epoch": 0.01,
726
+ "eval_ted_trans_nl-en_accuracy": 0.7532021898001414,
727
+ "eval_ted_trans_nl-en_loss": 1.0654296875,
728
+ "eval_ted_trans_nl-en_runtime": 7.9186,
729
+ "eval_ted_trans_nl-en_samples_per_second": 97.365,
730
+ "eval_ted_trans_nl-en_steps_per_second": 1.137,
731
+ "step": 500
732
+ },
733
+ {
734
+ "epoch": 0.01,
735
+ "eval_ted_trans_en-ja_accuracy": 0.6662950575994054,
736
+ "eval_ted_trans_en-ja_loss": 1.3916015625,
737
+ "eval_ted_trans_en-ja_runtime": 9.7107,
738
+ "eval_ted_trans_en-ja_samples_per_second": 82.486,
739
+ "eval_ted_trans_en-ja_steps_per_second": 0.927,
740
+ "step": 500
741
+ },
742
+ {
743
+ "epoch": 0.01,
744
+ "eval_ted_trans_en-es_accuracy": 0.7895431674388482,
745
+ "eval_ted_trans_en-es_loss": 0.87646484375,
746
+ "eval_ted_trans_en-es_runtime": 9.3046,
747
+ "eval_ted_trans_en-es_samples_per_second": 88.774,
748
+ "eval_ted_trans_en-es_steps_per_second": 0.967,
749
+ "step": 500
750
+ },
751
+ {
752
+ "epoch": 0.01,
753
+ "eval_ted_trans_en-ms_accuracy": 0.692143238909674,
754
+ "eval_ted_trans_en-ms_loss": 1.36328125,
755
+ "eval_ted_trans_en-ms_runtime": 1.0241,
756
+ "eval_ted_trans_en-ms_samples_per_second": 41.011,
757
+ "eval_ted_trans_en-ms_steps_per_second": 0.976,
758
+ "step": 500
759
+ },
760
+ {
761
+ "epoch": 0.01,
762
+ "eval_xsum_accuracy": 0.621155930594,
763
+ "eval_xsum_loss": 1.4501953125,
764
+ "eval_xsum_runtime": 440.7691,
765
+ "eval_xsum_samples_per_second": 92.586,
766
+ "eval_xsum_steps_per_second": 0.966,
767
+ "step": 500
768
+ },
769
+ {
770
+ "epoch": 0.01,
771
+ "eval_cnn_dailymail_accuracy": 0.6818918043407839,
772
+ "eval_cnn_dailymail_loss": NaN,
773
+ "eval_cnn_dailymail_runtime": 631.5892,
774
+ "eval_cnn_dailymail_samples_per_second": 90.918,
775
+ "eval_cnn_dailymail_steps_per_second": 0.948,
776
+ "step": 500
777
+ },
778
+ {
779
+ "epoch": 0.01,
780
+ "eval_multi_news_accuracy": 0.5512987425377873,
781
+ "eval_multi_news_loss": NaN,
782
+ "eval_multi_news_runtime": 102.5343,
783
+ "eval_multi_news_samples_per_second": 87.727,
784
+ "eval_multi_news_steps_per_second": 0.917,
785
+ "step": 500
786
+ },
787
+ {
788
+ "epoch": 0.01,
789
+ "eval_tldr_news_accuracy": 0.5479638860152356,
790
+ "eval_tldr_news_loss": 2.09375,
791
+ "eval_tldr_news_runtime": 7.6366,
792
+ "eval_tldr_news_samples_per_second": 186.994,
793
+ "eval_tldr_news_steps_per_second": 1.964,
794
+ "step": 500
795
+ },
796
+ {
797
+ "epoch": 0.01,
798
+ "eval_scitldr_accuracy": 0.4991896272285251,
799
+ "eval_scitldr_loss": NaN,
800
+ "eval_scitldr_runtime": 5.9643,
801
+ "eval_scitldr_samples_per_second": 66.899,
802
+ "eval_scitldr_steps_per_second": 0.838,
803
+ "step": 500
804
+ },
805
+ {
806
+ "epoch": 0.01,
807
+ "eval_samsum_accuracy": 0.6392542821992997,
808
+ "eval_samsum_loss": 1.3603515625,
809
+ "eval_samsum_runtime": 31.1036,
810
+ "eval_samsum_samples_per_second": 94.748,
811
+ "eval_samsum_steps_per_second": 0.997,
812
+ "step": 500
813
+ },
814
+ {
815
+ "epoch": 0.01,
816
+ "eval_debate_sum_accuracy": 0.9375640253883767,
817
+ "eval_debate_sum_loss": 0.34521484375,
818
+ "eval_debate_sum_runtime": 548.5555,
819
+ "eval_debate_sum_samples_per_second": 87.71,
820
+ "eval_debate_sum_steps_per_second": 0.915,
821
+ "step": 500
822
+ },
823
+ {
824
+ "epoch": 0.01,
825
+ "eval_billsum_accuracy": 0.6806867345609693,
826
+ "eval_billsum_loss": 1.3427734375,
827
+ "eval_billsum_runtime": 43.496,
828
+ "eval_billsum_samples_per_second": 87.134,
829
+ "eval_billsum_steps_per_second": 0.92,
830
+ "step": 500
831
+ },
832
+ {
833
+ "epoch": 0.01,
834
+ "eval_wmt2019_zh-en_accuracy": 0.6670238429829493,
835
+ "eval_wmt2019_zh-en_loss": 1.453125,
836
+ "eval_wmt2019_zh-en_runtime": 28.9371,
837
+ "eval_wmt2019_zh-en_samples_per_second": 137.574,
838
+ "eval_wmt2019_zh-en_steps_per_second": 1.451,
839
+ "step": 500
840
+ },
841
+ {
842
+ "epoch": 0.01,
843
+ "eval_wmt2019_ru-en_accuracy": 0.7587101830765136,
844
+ "eval_wmt2019_ru-en_loss": 0.92724609375,
845
+ "eval_wmt2019_ru-en_runtime": 23.7381,
846
+ "eval_wmt2019_ru-en_samples_per_second": 126.379,
847
+ "eval_wmt2019_ru-en_steps_per_second": 1.348,
848
+ "step": 500
849
+ },
850
+ {
851
+ "epoch": 0.01,
852
+ "eval_wmt2019_de-en_accuracy": 0.7675478121558026,
853
+ "eval_wmt2019_de-en_loss": 0.90478515625,
854
+ "eval_wmt2019_de-en_runtime": 16.2264,
855
+ "eval_wmt2019_de-en_samples_per_second": 184.76,
856
+ "eval_wmt2019_de-en_steps_per_second": 1.972,
857
+ "step": 500
858
+ },
859
+ {
860
+ "epoch": 0.01,
861
+ "eval_wmt2019_fr-de_accuracy": 0.7500888456249324,
862
+ "eval_wmt2019_fr-de_loss": 0.99560546875,
863
+ "eval_wmt2019_fr-de_runtime": 11.5712,
864
+ "eval_wmt2019_fr-de_samples_per_second": 130.669,
865
+ "eval_wmt2019_fr-de_steps_per_second": 1.383,
866
+ "step": 500
867
+ },
868
+ {
869
+ "epoch": 0.01,
870
+ "eval_essay_instruction_accuracy": 0.6002366052672313,
871
+ "eval_essay_instruction_loss": 1.9189453125,
872
+ "eval_essay_instruction_runtime": 8.0794,
873
+ "eval_essay_instruction_samples_per_second": 51.118,
874
+ "eval_essay_instruction_steps_per_second": 0.619,
875
+ "step": 500
876
+ },
877
+ {
878
+ "epoch": 0.01,
879
+ "eval_reddit_eli5_accuracy": 0.46082089893518746,
880
+ "eval_reddit_eli5_loss": 2.4296875,
881
+ "eval_reddit_eli5_runtime": 602.6271,
882
+ "eval_reddit_eli5_samples_per_second": 90.482,
883
+ "eval_reddit_eli5_steps_per_second": 0.943,
884
+ "step": 500
885
+ },
886
+ {
887
+ "epoch": 0.01,
888
+ "eval_reddit_askh_accuracy": 0.46347532552175574,
889
+ "eval_reddit_askh_loss": 2.52734375,
890
+ "eval_reddit_askh_runtime": 245.7671,
891
+ "eval_reddit_askh_samples_per_second": 80.178,
892
+ "eval_reddit_askh_steps_per_second": 0.838,
893
+ "step": 500
894
+ },
895
+ {
896
+ "epoch": 0.01,
897
+ "eval_reddit_asks_accuracy": 0.47150193020881753,
898
+ "eval_reddit_asks_loss": 2.38671875,
899
+ "eval_reddit_asks_runtime": 320.7509,
900
+ "eval_reddit_asks_samples_per_second": 82.17,
901
+ "eval_reddit_asks_steps_per_second": 0.857,
902
+ "step": 500
903
+ },
904
+ {
905
+ "epoch": 0.02,
906
+ "learning_rate": 4.5083465988888945e-06,
907
+ "loss": 1.5195,
908
+ "step": 510
909
+ },
910
+ {
911
+ "epoch": 0.02,
912
+ "learning_rate": 4.5224842384899045e-06,
913
+ "loss": 1.492,
914
+ "step": 520
915
+ },
916
+ {
917
+ "epoch": 0.02,
918
+ "learning_rate": 4.5363510253542444e-06,
919
+ "loss": 1.5302,
920
+ "step": 530
921
+ },
922
+ {
923
+ "epoch": 0.02,
924
+ "learning_rate": 4.549957142832593e-06,
925
+ "loss": 1.5267,
926
+ "step": 540
927
+ },
928
+ {
929
+ "epoch": 0.02,
930
+ "learning_rate": 4.563312210555719e-06,
931
+ "loss": 1.565,
932
+ "step": 550
933
+ },
934
+ {
935
+ "epoch": 0.02,
936
+ "learning_rate": 4.576425325289549e-06,
937
+ "loss": 1.6208,
938
+ "step": 560
939
+ },
940
+ {
941
+ "epoch": 0.02,
942
+ "learning_rate": 4.589305098154845e-06,
943
+ "loss": 1.6341,
944
+ "step": 570
945
+ },
946
+ {
947
+ "epoch": 0.02,
948
+ "learning_rate": 4.601959688592886e-06,
949
+ "loss": 1.5639,
950
+ "step": 580
951
+ },
952
+ {
953
+ "epoch": 0.02,
954
+ "learning_rate": 4.614396835412691e-06,
955
+ "loss": 1.6218,
956
+ "step": 590
957
+ },
958
+ {
959
+ "epoch": 0.02,
960
+ "learning_rate": 4.626623885215616e-06,
961
+ "loss": 1.5995,
962
+ "step": 600
963
+ },
964
+ {
965
+ "epoch": 0.02,
966
+ "learning_rate": 4.638647818458763e-06,
967
+ "loss": 1.6176,
968
+ "step": 610
969
+ },
970
+ {
971
+ "epoch": 0.02,
972
+ "learning_rate": 4.650475273388737e-06,
973
+ "loss": 1.5944,
974
+ "step": 620
975
+ },
976
+ {
977
+ "epoch": 0.02,
978
+ "learning_rate": 4.662112568051194e-06,
979
+ "loss": 1.6074,
980
+ "step": 630
981
+ },
982
+ {
983
+ "epoch": 0.02,
984
+ "learning_rate": 4.673565720558918e-06,
985
+ "loss": 1.5783,
986
+ "step": 640
987
+ },
988
+ {
989
+ "epoch": 0.02,
990
+ "learning_rate": 4.6848404677811685e-06,
991
+ "loss": 1.5135,
992
+ "step": 650
993
+ },
994
+ {
995
+ "epoch": 0.02,
996
+ "learning_rate": 4.695942282599635e-06,
997
+ "loss": 1.6396,
998
+ "step": 660
999
+ },
1000
+ {
1001
+ "epoch": 0.02,
1002
+ "learning_rate": 4.706876389860915e-06,
1003
+ "loss": 1.6053,
1004
+ "step": 670
1005
+ },
1006
+ {
1007
+ "epoch": 0.02,
1008
+ "learning_rate": 4.717647781141908e-06,
1009
+ "loss": 1.5982,
1010
+ "step": 680
1011
+ },
1012
+ {
1013
+ "epoch": 0.02,
1014
+ "learning_rate": 4.7282612284325845e-06,
1015
+ "loss": 1.5361,
1016
+ "step": 690
1017
+ },
1018
+ {
1019
+ "epoch": 0.02,
1020
+ "learning_rate": 4.738721296830016e-06,
1021
+ "loss": 1.5127,
1022
+ "step": 700
1023
+ },
1024
+ {
1025
+ "epoch": 0.02,
1026
+ "learning_rate": 4.749032356328167e-06,
1027
+ "loss": 1.4852,
1028
+ "step": 710
1029
+ },
1030
+ {
1031
+ "epoch": 0.02,
1032
+ "learning_rate": 4.759198592779668e-06,
1033
+ "loss": 1.5432,
1034
+ "step": 720
1035
+ },
1036
+ {
1037
+ "epoch": 0.02,
1038
+ "learning_rate": 4.769224018098397e-06,
1039
+ "loss": 1.5425,
1040
+ "step": 730
1041
+ },
1042
+ {
1043
+ "epoch": 0.02,
1044
+ "learning_rate": 4.7791124797650865e-06,
1045
+ "loss": 1.493,
1046
+ "step": 740
1047
+ },
1048
+ {
1049
+ "epoch": 0.02,
1050
+ "learning_rate": 4.788867669692332e-06,
1051
+ "loss": 1.5065,
1052
+ "step": 750
1053
+ },
1054
+ {
1055
+ "epoch": 0.02,
1056
+ "eval_gsm8k_hard_accuracy": 0.9174097145881682,
1057
+ "eval_gsm8k_hard_loss": 0.366455078125,
1058
+ "eval_gsm8k_hard_runtime": 6.7984,
1059
+ "eval_gsm8k_hard_samples_per_second": 38.833,
1060
+ "eval_gsm8k_hard_steps_per_second": 0.441,
1061
+ "step": 750
1062
+ },
1063
+ {
1064
+ "epoch": 0.02,
1065
+ "eval_webgpt_accuracy": 0.5023221414992414,
1066
+ "eval_webgpt_loss": 2.181640625,
1067
+ "eval_webgpt_runtime": 39.4537,
1068
+ "eval_webgpt_samples_per_second": 99.256,
1069
+ "eval_webgpt_steps_per_second": 1.039,
1070
+ "step": 750
1071
+ },
1072
+ {
1073
+ "epoch": 0.02,
1074
+ "eval_squad_v2_accuracy": 0.8977014895925817,
1075
+ "eval_squad_v2_loss": 0.331787109375,
1076
+ "eval_squad_v2_runtime": 214.9281,
1077
+ "eval_squad_v2_samples_per_second": 121.268,
1078
+ "eval_squad_v2_steps_per_second": 1.266,
1079
+ "step": 750
1080
+ },
1081
+ {
1082
+ "epoch": 0.02,
1083
+ "eval_adversarial_qa_accuracy": 0.8063639891346527,
1084
+ "eval_adversarial_qa_loss": 0.8232421875,
1085
+ "eval_adversarial_qa_runtime": 51.9182,
1086
+ "eval_adversarial_qa_samples_per_second": 115.567,
1087
+ "eval_adversarial_qa_steps_per_second": 1.213,
1088
+ "step": 750
1089
+ },
1090
+ {
1091
+ "epoch": 0.02,
1092
+ "eval_private_tuning_accuracy": 0.6775308778615678,
1093
+ "eval_private_tuning_loss": 1.1962890625,
1094
+ "eval_private_tuning_runtime": 149.5702,
1095
+ "eval_private_tuning_samples_per_second": 141.592,
1096
+ "eval_private_tuning_steps_per_second": 1.478,
1097
+ "step": 750
1098
+ },
1099
+ {
1100
+ "epoch": 0.02,
1101
+ "eval_oa_translated_accuracy": 0.6986534506008611,
1102
+ "eval_oa_translated_loss": 1.22265625,
1103
+ "eval_oa_translated_runtime": 1324.5514,
1104
+ "eval_oa_translated_samples_per_second": 89.655,
1105
+ "eval_oa_translated_steps_per_second": 0.935,
1106
+ "step": 750
1107
+ },
1108
+ {
1109
+ "epoch": 0.02,
1110
+ "eval_prosocial_dialogue_accuracy": 0.5327101026505052,
1111
+ "eval_prosocial_dialogue_loss": 1.7802734375,
1112
+ "eval_prosocial_dialogue_runtime": 70.7166,
1113
+ "eval_prosocial_dialogue_samples_per_second": 381.565,
1114
+ "eval_prosocial_dialogue_steps_per_second": 3.988,
1115
+ "step": 750
1116
+ },
1117
+ {
1118
+ "epoch": 0.02,
1119
+ "eval_math_qa_accuracy": 0.5798378605476227,
1120
+ "eval_math_qa_loss": 1.826171875,
1121
+ "eval_math_qa_runtime": 44.6748,
1122
+ "eval_math_qa_samples_per_second": 133.588,
1123
+ "eval_math_qa_steps_per_second": 1.41,
1124
+ "step": 750
1125
+ },
1126
+ {
1127
+ "epoch": 0.02,
1128
+ "eval_wikihow_accuracy": 0.6193731798640966,
1129
+ "eval_wikihow_loss": 1.802734375,
1130
+ "eval_wikihow_runtime": 16.8626,
1131
+ "eval_wikihow_samples_per_second": 135.981,
1132
+ "eval_wikihow_steps_per_second": 1.423,
1133
+ "step": 750
1134
+ },
1135
+ {
1136
+ "epoch": 0.02,
1137
+ "eval_joke_accuracy": 0.5020849128127369,
1138
+ "eval_joke_loss": 2.1640625,
1139
+ "eval_joke_runtime": 1.3597,
1140
+ "eval_joke_samples_per_second": 55.896,
1141
+ "eval_joke_steps_per_second": 0.735,
1142
+ "step": 750
1143
+ },
1144
+ {
1145
+ "epoch": 0.02,
1146
+ "eval_gsm8k_accuracy": 0.760008955934006,
1147
+ "eval_gsm8k_loss": 0.9189453125,
1148
+ "eval_gsm8k_runtime": 12.0443,
1149
+ "eval_gsm8k_samples_per_second": 124.126,
1150
+ "eval_gsm8k_steps_per_second": 1.328,
1151
+ "step": 750
1152
+ },
1153
+ {
1154
+ "epoch": 0.02,
1155
+ "eval_ted_trans_en-hi_accuracy": 0.6714796661809511,
1156
+ "eval_ted_trans_en-hi_loss": 1.2548828125,
1157
+ "eval_ted_trans_en-hi_runtime": 2.3695,
1158
+ "eval_ted_trans_en-hi_samples_per_second": 43.47,
1159
+ "eval_ted_trans_en-hi_steps_per_second": 0.844,
1160
+ "step": 750
1161
+ },
1162
+ {
1163
+ "epoch": 0.02,
1164
+ "eval_ted_trans_de-ja_accuracy": 0.6580367185861629,
1165
+ "eval_ted_trans_de-ja_loss": 1.466796875,
1166
+ "eval_ted_trans_de-ja_runtime": 9.4824,
1167
+ "eval_ted_trans_de-ja_samples_per_second": 75.72,
1168
+ "eval_ted_trans_de-ja_steps_per_second": 0.844,
1169
+ "step": 750
1170
+ },
1171
+ {
1172
+ "epoch": 0.02,
1173
+ "eval_ted_trans_nl-en_accuracy": 0.749224515991015,
1174
+ "eval_ted_trans_nl-en_loss": 1.080078125,
1175
+ "eval_ted_trans_nl-en_runtime": 8.4451,
1176
+ "eval_ted_trans_nl-en_samples_per_second": 91.296,
1177
+ "eval_ted_trans_nl-en_steps_per_second": 1.066,
1178
+ "step": 750
1179
+ },
1180
+ {
1181
+ "epoch": 0.02,
1182
+ "eval_ted_trans_en-ja_accuracy": 0.6621738060068931,
1183
+ "eval_ted_trans_en-ja_loss": 1.384765625,
1184
+ "eval_ted_trans_en-ja_runtime": 10.0893,
1185
+ "eval_ted_trans_en-ja_samples_per_second": 79.391,
1186
+ "eval_ted_trans_en-ja_steps_per_second": 0.892,
1187
+ "step": 750
1188
+ },
1189
+ {
1190
+ "epoch": 0.02,
1191
+ "eval_ted_trans_en-es_accuracy": 0.793457991028678,
1192
+ "eval_ted_trans_en-es_loss": 0.85205078125,
1193
+ "eval_ted_trans_en-es_runtime": 7.1771,
1194
+ "eval_ted_trans_en-es_samples_per_second": 115.088,
1195
+ "eval_ted_trans_en-es_steps_per_second": 1.254,
1196
+ "step": 750
1197
+ },
1198
+ {
1199
+ "epoch": 0.02,
1200
+ "eval_ted_trans_en-ms_accuracy": 0.6782511210762332,
1201
+ "eval_ted_trans_en-ms_loss": 1.4208984375,
1202
+ "eval_ted_trans_en-ms_runtime": 2.0842,
1203
+ "eval_ted_trans_en-ms_samples_per_second": 20.151,
1204
+ "eval_ted_trans_en-ms_steps_per_second": 0.48,
1205
+ "step": 750
1206
+ },
1207
+ {
1208
+ "epoch": 0.02,
1209
+ "eval_xsum_accuracy": 0.6225130561751453,
1210
+ "eval_xsum_loss": 1.4462890625,
1211
+ "eval_xsum_runtime": 439.444,
1212
+ "eval_xsum_samples_per_second": 92.865,
1213
+ "eval_xsum_steps_per_second": 0.969,
1214
+ "step": 750
1215
+ },
1216
+ {
1217
+ "epoch": 0.02,
1218
+ "eval_cnn_dailymail_accuracy": 0.6778638530242029,
1219
+ "eval_cnn_dailymail_loss": NaN,
1220
+ "eval_cnn_dailymail_runtime": 633.9568,
1221
+ "eval_cnn_dailymail_samples_per_second": 90.579,
1222
+ "eval_cnn_dailymail_steps_per_second": 0.945,
1223
+ "step": 750
1224
+ },
1225
+ {
1226
+ "epoch": 0.02,
1227
+ "eval_multi_news_accuracy": 0.5553791439095643,
1228
+ "eval_multi_news_loss": NaN,
1229
+ "eval_multi_news_runtime": 103.0316,
1230
+ "eval_multi_news_samples_per_second": 87.303,
1231
+ "eval_multi_news_steps_per_second": 0.912,
1232
+ "step": 750
1233
+ },
1234
+ {
1235
+ "epoch": 0.02,
1236
+ "eval_tldr_news_accuracy": 0.5591084360011286,
1237
+ "eval_tldr_news_loss": 1.9892578125,
1238
+ "eval_tldr_news_runtime": 8.3367,
1239
+ "eval_tldr_news_samples_per_second": 171.29,
1240
+ "eval_tldr_news_steps_per_second": 1.799,
1241
+ "step": 750
1242
+ },
1243
+ {
1244
+ "epoch": 0.02,
1245
+ "eval_scitldr_accuracy": 0.49270664505672607,
1246
+ "eval_scitldr_loss": NaN,
1247
+ "eval_scitldr_runtime": 5.8517,
1248
+ "eval_scitldr_samples_per_second": 68.186,
1249
+ "eval_scitldr_steps_per_second": 0.854,
1250
+ "step": 750
1251
+ },
1252
+ {
1253
+ "epoch": 0.02,
1254
+ "eval_samsum_accuracy": 0.6411875245035082,
1255
+ "eval_samsum_loss": 1.32421875,
1256
+ "eval_samsum_runtime": 32.2591,
1257
+ "eval_samsum_samples_per_second": 91.354,
1258
+ "eval_samsum_steps_per_second": 0.961,
1259
+ "step": 750
1260
+ },
1261
+ {
1262
+ "epoch": 0.02,
1263
+ "eval_debate_sum_accuracy": 0.9381249710591028,
1264
+ "eval_debate_sum_loss": 0.337646484375,
1265
+ "eval_debate_sum_runtime": 548.1225,
1266
+ "eval_debate_sum_samples_per_second": 87.78,
1267
+ "eval_debate_sum_steps_per_second": 0.916,
1268
+ "step": 750
1269
+ },
1270
+ {
1271
+ "epoch": 0.02,
1272
+ "eval_billsum_accuracy": 0.6810246806233696,
1273
+ "eval_billsum_loss": 1.3359375,
1274
+ "eval_billsum_runtime": 50.0247,
1275
+ "eval_billsum_samples_per_second": 75.763,
1276
+ "eval_billsum_steps_per_second": 0.8,
1277
+ "step": 750
1278
+ },
1279
+ {
1280
+ "epoch": 0.02,
1281
+ "eval_wmt2019_zh-en_accuracy": 0.6683125468349724,
1282
+ "eval_wmt2019_zh-en_loss": 1.451171875,
1283
+ "eval_wmt2019_zh-en_runtime": 27.2087,
1284
+ "eval_wmt2019_zh-en_samples_per_second": 146.313,
1285
+ "eval_wmt2019_zh-en_steps_per_second": 1.544,
1286
+ "step": 750
1287
+ },
1288
+ {
1289
+ "epoch": 0.02,
1290
+ "eval_wmt2019_ru-en_accuracy": 0.755552089368213,
1291
+ "eval_wmt2019_ru-en_loss": 0.94091796875,
1292
+ "eval_wmt2019_ru-en_runtime": 20.7954,
1293
+ "eval_wmt2019_ru-en_samples_per_second": 144.262,
1294
+ "eval_wmt2019_ru-en_steps_per_second": 1.539,
1295
+ "step": 750
1296
+ },
1297
+ {
1298
+ "epoch": 0.02,
1299
+ "eval_wmt2019_de-en_accuracy": 0.7641599453590333,
1300
+ "eval_wmt2019_de-en_loss": 0.9228515625,
1301
+ "eval_wmt2019_de-en_runtime": 15.5528,
1302
+ "eval_wmt2019_de-en_samples_per_second": 192.762,
1303
+ "eval_wmt2019_de-en_steps_per_second": 2.058,
1304
+ "step": 750
1305
+ },
1306
+ {
1307
+ "epoch": 0.02,
1308
+ "eval_wmt2019_fr-de_accuracy": 0.7474449624849476,
1309
+ "eval_wmt2019_fr-de_loss": 1.00390625,
1310
+ "eval_wmt2019_fr-de_runtime": 11.5093,
1311
+ "eval_wmt2019_fr-de_samples_per_second": 131.372,
1312
+ "eval_wmt2019_fr-de_steps_per_second": 1.39,
1313
+ "step": 750
1314
+ },
1315
+ {
1316
+ "epoch": 0.02,
1317
+ "eval_essay_instruction_accuracy": 0.6032218119098689,
1318
+ "eval_essay_instruction_loss": 1.904296875,
1319
+ "eval_essay_instruction_runtime": 7.606,
1320
+ "eval_essay_instruction_samples_per_second": 54.299,
1321
+ "eval_essay_instruction_steps_per_second": 0.657,
1322
+ "step": 750
1323
+ },
1324
+ {
1325
+ "epoch": 0.02,
1326
+ "eval_reddit_eli5_accuracy": 0.4612608360817972,
1327
+ "eval_reddit_eli5_loss": 2.431640625,
1328
+ "eval_reddit_eli5_runtime": 597.1988,
1329
+ "eval_reddit_eli5_samples_per_second": 91.305,
1330
+ "eval_reddit_eli5_steps_per_second": 0.951,
1331
+ "step": 750
1332
+ },
1333
+ {
1334
+ "epoch": 0.02,
1335
+ "eval_reddit_askh_accuracy": 0.46371300245115404,
1336
+ "eval_reddit_askh_loss": 2.525390625,
1337
+ "eval_reddit_askh_runtime": 253.0373,
1338
+ "eval_reddit_askh_samples_per_second": 77.874,
1339
+ "eval_reddit_askh_steps_per_second": 0.814,
1340
+ "step": 750
1341
+ },
1342
+ {
1343
+ "epoch": 0.02,
1344
+ "eval_reddit_asks_accuracy": 0.47195547000535765,
1345
+ "eval_reddit_asks_loss": 2.388671875,
1346
+ "eval_reddit_asks_runtime": 304.1555,
1347
+ "eval_reddit_asks_samples_per_second": 86.653,
1348
+ "eval_reddit_asks_steps_per_second": 0.904,
1349
+ "step": 750
1350
+ },
1351
+ {
1352
+ "epoch": 0.02,
1353
+ "learning_rate": 4.798493132500121e-06,
1354
+ "loss": 1.5526,
1355
+ "step": 760
1356
+ },
1357
+ {
1358
+ "epoch": 0.02,
1359
+ "learning_rate": 4.8079922732483016e-06,
1360
+ "loss": 1.4845,
1361
+ "step": 770
1362
+ },
1363
+ {
1364
+ "epoch": 0.02,
1365
+ "learning_rate": 4.817368364668191e-06,
1366
+ "loss": 1.5351,
1367
+ "step": 780
1368
+ },
1369
+ {
1370
+ "epoch": 0.02,
1371
+ "learning_rate": 4.8266245539317745e-06,
1372
+ "loss": 1.5942,
1373
+ "step": 790
1374
+ },
1375
+ {
1376
+ "epoch": 0.02,
1377
+ "learning_rate": 4.835763868993521e-06,
1378
+ "loss": 1.4886,
1379
+ "step": 800
1380
+ },
1381
+ {
1382
+ "epoch": 0.02,
1383
+ "learning_rate": 4.844789224536785e-06,
1384
+ "loss": 1.5645,
1385
+ "step": 810
1386
+ },
1387
+ {
1388
+ "epoch": 0.02,
1389
+ "learning_rate": 4.853703427554027e-06,
1390
+ "loss": 1.5099,
1391
+ "step": 820
1392
+ },
1393
+ {
1394
+ "epoch": 0.02,
1395
+ "learning_rate": 4.862509182587578e-06,
1396
+ "loss": 1.619,
1397
+ "step": 830
1398
+ },
1399
+ {
1400
+ "epoch": 0.02,
1401
+ "learning_rate": 4.871209096655434e-06,
1402
+ "loss": 1.542,
1403
+ "step": 840
1404
+ },
1405
+ {
1406
+ "epoch": 0.03,
1407
+ "learning_rate": 4.879805683884512e-06,
1408
+ "loss": 1.5254,
1409
+ "step": 850
1410
+ },
1411
+ {
1412
+ "epoch": 0.03,
1413
+ "learning_rate": 4.888301369871998e-06,
1414
+ "loss": 1.5427,
1415
+ "step": 860
1416
+ },
1417
+ {
1418
+ "epoch": 0.03,
1419
+ "learning_rate": 4.8966984957936845e-06,
1420
+ "loss": 1.5403,
1421
+ "step": 870
1422
+ },
1423
+ {
1424
+ "epoch": 0.03,
1425
+ "learning_rate": 4.904999322276735e-06,
1426
+ "loss": 1.5848,
1427
+ "step": 880
1428
+ },
1429
+ {
1430
+ "epoch": 0.03,
1431
+ "learning_rate": 4.913206033052878e-06,
1432
+ "loss": 1.5205,
1433
+ "step": 890
1434
+ },
1435
+ {
1436
+ "epoch": 0.03,
1437
+ "learning_rate": 4.921320738406821e-06,
1438
+ "loss": 1.5359,
1439
+ "step": 900
1440
+ },
1441
+ {
1442
+ "epoch": 0.03,
1443
+ "learning_rate": 4.929345478433492e-06,
1444
+ "loss": 1.5631,
1445
+ "step": 910
1446
+ },
1447
+ {
1448
+ "epoch": 0.03,
1449
+ "learning_rate": 4.937282226116702e-06,
1450
+ "loss": 1.5928,
1451
+ "step": 920
1452
+ },
1453
+ {
1454
+ "epoch": 0.03,
1455
+ "learning_rate": 4.945132890240829e-06,
1456
+ "loss": 1.4707,
1457
+ "step": 930
1458
+ },
1459
+ {
1460
+ "epoch": 0.03,
1461
+ "learning_rate": 4.952899318146298e-06,
1462
+ "loss": 1.5279,
1463
+ "step": 940
1464
+ },
1465
+ {
1466
+ "epoch": 0.03,
1467
+ "learning_rate": 4.96058329833879e-06,
1468
+ "loss": 1.5411,
1469
+ "step": 950
1470
+ },
1471
+ {
1472
+ "epoch": 0.03,
1473
+ "learning_rate": 4.968186562961406e-06,
1474
+ "loss": 1.6029,
1475
+ "step": 960
1476
+ },
1477
+ {
1478
+ "epoch": 0.03,
1479
+ "learning_rate": 4.975710790138337e-06,
1480
+ "loss": 1.648,
1481
+ "step": 970
1482
+ },
1483
+ {
1484
+ "epoch": 0.03,
1485
+ "learning_rate": 4.9831576061979556e-06,
1486
+ "loss": 1.5799,
1487
+ "step": 980
1488
+ },
1489
+ {
1490
+ "epoch": 0.03,
1491
+ "learning_rate": 4.990528587782728e-06,
1492
+ "loss": 1.5592,
1493
+ "step": 990
1494
+ },
1495
+ {
1496
+ "epoch": 0.03,
1497
+ "learning_rate": 4.99782526385276e-06,
1498
+ "loss": 1.6317,
1499
+ "step": 1000
1500
+ },
1501
+ {
1502
+ "epoch": 0.03,
1503
+ "eval_gsm8k_hard_accuracy": 0.9242294694841415,
1504
+ "eval_gsm8k_hard_loss": 0.337158203125,
1505
+ "eval_gsm8k_hard_runtime": 4.4214,
1506
+ "eval_gsm8k_hard_samples_per_second": 59.709,
1507
+ "eval_gsm8k_hard_steps_per_second": 0.679,
1508
+ "step": 1000
1509
+ },
1510
+ {
1511
+ "epoch": 0.03,
1512
+ "eval_webgpt_accuracy": 0.5016719087887306,
1513
+ "eval_webgpt_loss": 2.181640625,
1514
+ "eval_webgpt_runtime": 36.3649,
1515
+ "eval_webgpt_samples_per_second": 107.686,
1516
+ "eval_webgpt_steps_per_second": 1.127,
1517
+ "step": 1000
1518
+ },
1519
+ {
1520
+ "epoch": 0.03,
1521
+ "eval_squad_v2_accuracy": 0.9092151805972463,
1522
+ "eval_squad_v2_loss": 0.35546875,
1523
+ "eval_squad_v2_runtime": 216.2111,
1524
+ "eval_squad_v2_samples_per_second": 120.549,
1525
+ "eval_squad_v2_steps_per_second": 1.258,
1526
+ "step": 1000
1527
+ },
1528
+ {
1529
+ "epoch": 0.03,
1530
+ "eval_adversarial_qa_accuracy": 0.8333781081161756,
1531
+ "eval_adversarial_qa_loss": 0.83642578125,
1532
+ "eval_adversarial_qa_runtime": 52.2959,
1533
+ "eval_adversarial_qa_samples_per_second": 114.732,
1534
+ "eval_adversarial_qa_steps_per_second": 1.205,
1535
+ "step": 1000
1536
+ },
1537
+ {
1538
+ "epoch": 0.03,
1539
+ "eval_private_tuning_accuracy": 0.6788522917969135,
1540
+ "eval_private_tuning_loss": 1.1845703125,
1541
+ "eval_private_tuning_runtime": 145.6236,
1542
+ "eval_private_tuning_samples_per_second": 145.43,
1543
+ "eval_private_tuning_steps_per_second": 1.518,
1544
+ "step": 1000
1545
+ },
1546
+ {
1547
+ "epoch": 0.03,
1548
+ "eval_oa_translated_accuracy": 0.7015835515263078,
1549
+ "eval_oa_translated_loss": 1.208984375,
1550
+ "eval_oa_translated_runtime": 1331.5436,
1551
+ "eval_oa_translated_samples_per_second": 89.184,
1552
+ "eval_oa_translated_steps_per_second": 0.93,
1553
+ "step": 1000
1554
+ },
1555
+ {
1556
+ "epoch": 0.03,
1557
+ "eval_prosocial_dialogue_accuracy": 0.5440172516743936,
1558
+ "eval_prosocial_dialogue_loss": 1.7470703125,
1559
+ "eval_prosocial_dialogue_runtime": 66.8792,
1560
+ "eval_prosocial_dialogue_samples_per_second": 403.459,
1561
+ "eval_prosocial_dialogue_steps_per_second": 4.217,
1562
+ "step": 1000
1563
+ },
1564
+ {
1565
+ "epoch": 0.03,
1566
+ "eval_math_qa_accuracy": 0.5903696634283728,
1567
+ "eval_math_qa_loss": 1.7734375,
1568
+ "eval_math_qa_runtime": 43.3917,
1569
+ "eval_math_qa_samples_per_second": 137.538,
1570
+ "eval_math_qa_steps_per_second": 1.452,
1571
+ "step": 1000
1572
+ },
1573
+ {
1574
+ "epoch": 0.03,
1575
+ "eval_wikihow_accuracy": 0.6181528220773818,
1576
+ "eval_wikihow_loss": 1.79296875,
1577
+ "eval_wikihow_runtime": 16.8686,
1578
+ "eval_wikihow_samples_per_second": 135.933,
1579
+ "eval_wikihow_steps_per_second": 1.423,
1580
+ "step": 1000
1581
+ },
1582
+ {
1583
+ "epoch": 0.03,
1584
+ "eval_joke_accuracy": 0.5162054586808188,
1585
+ "eval_joke_loss": 2.095703125,
1586
+ "eval_joke_runtime": 1.1499,
1587
+ "eval_joke_samples_per_second": 66.094,
1588
+ "eval_joke_steps_per_second": 0.87,
1589
+ "step": 1000
1590
+ },
1591
+ {
1592
+ "epoch": 0.03,
1593
+ "eval_gsm8k_accuracy": 0.7709449909740977,
1594
+ "eval_gsm8k_loss": 0.8671875,
1595
+ "eval_gsm8k_runtime": 11.5578,
1596
+ "eval_gsm8k_samples_per_second": 129.35,
1597
+ "eval_gsm8k_steps_per_second": 1.384,
1598
+ "step": 1000
1599
+ },
1600
+ {
1601
+ "epoch": 0.03,
1602
+ "eval_ted_trans_en-hi_accuracy": 0.6727249123718032,
1603
+ "eval_ted_trans_en-hi_loss": 1.2490234375,
1604
+ "eval_ted_trans_en-hi_runtime": 3.3114,
1605
+ "eval_ted_trans_en-hi_samples_per_second": 31.105,
1606
+ "eval_ted_trans_en-hi_steps_per_second": 0.604,
1607
+ "step": 1000
1608
+ },
1609
+ {
1610
+ "epoch": 0.03,
1611
+ "eval_ted_trans_de-ja_accuracy": 0.6605206483545547,
1612
+ "eval_ted_trans_de-ja_loss": 1.4599609375,
1613
+ "eval_ted_trans_de-ja_runtime": 8.583,
1614
+ "eval_ted_trans_de-ja_samples_per_second": 83.654,
1615
+ "eval_ted_trans_de-ja_steps_per_second": 0.932,
1616
+ "step": 1000
1617
+ },
1618
+ {
1619
+ "epoch": 0.03,
1620
+ "eval_ted_trans_nl-en_accuracy": 0.757177992835374,
1621
+ "eval_ted_trans_nl-en_loss": 1.0478515625,
1622
+ "eval_ted_trans_nl-en_runtime": 8.7612,
1623
+ "eval_ted_trans_nl-en_samples_per_second": 88.002,
1624
+ "eval_ted_trans_nl-en_steps_per_second": 1.027,
1625
+ "step": 1000
1626
+ },
1627
+ {
1628
+ "epoch": 0.03,
1629
+ "eval_ted_trans_en-ja_accuracy": 0.6644915715062534,
1630
+ "eval_ted_trans_en-ja_loss": 1.3798828125,
1631
+ "eval_ted_trans_en-ja_runtime": 9.6809,
1632
+ "eval_ted_trans_en-ja_samples_per_second": 82.74,
1633
+ "eval_ted_trans_en-ja_steps_per_second": 0.93,
1634
+ "step": 1000
1635
+ },
1636
+ {
1637
+ "epoch": 0.03,
1638
+ "eval_ted_trans_en-es_accuracy": 0.7831230683487865,
1639
+ "eval_ted_trans_en-es_loss": 0.89501953125,
1640
+ "eval_ted_trans_en-es_runtime": 8.1422,
1641
+ "eval_ted_trans_en-es_samples_per_second": 101.447,
1642
+ "eval_ted_trans_en-es_steps_per_second": 1.105,
1643
+ "step": 1000
1644
+ },
1645
+ {
1646
+ "epoch": 0.03,
1647
+ "eval_ted_trans_en-ms_accuracy": 0.6917040358744395,
1648
+ "eval_ted_trans_en-ms_loss": 1.3955078125,
1649
+ "eval_ted_trans_en-ms_runtime": 0.7332,
1650
+ "eval_ted_trans_en-ms_samples_per_second": 57.285,
1651
+ "eval_ted_trans_en-ms_steps_per_second": 1.364,
1652
+ "step": 1000
1653
+ },
1654
+ {
1655
+ "epoch": 0.03,
1656
+ "eval_xsum_accuracy": 0.6225837900623918,
1657
+ "eval_xsum_loss": 1.4453125,
1658
+ "eval_xsum_runtime": 443.13,
1659
+ "eval_xsum_samples_per_second": 92.093,
1660
+ "eval_xsum_steps_per_second": 0.961,
1661
+ "step": 1000
1662
+ },
1663
+ {
1664
+ "epoch": 0.03,
1665
+ "eval_cnn_dailymail_accuracy": 0.6811569253551761,
1666
+ "eval_cnn_dailymail_loss": NaN,
1667
+ "eval_cnn_dailymail_runtime": 634.279,
1668
+ "eval_cnn_dailymail_samples_per_second": 90.533,
1669
+ "eval_cnn_dailymail_steps_per_second": 0.944,
1670
+ "step": 1000
1671
+ },
1672
+ {
1673
+ "epoch": 0.03,
1674
+ "eval_multi_news_accuracy": 0.5572843896862695,
1675
+ "eval_multi_news_loss": NaN,
1676
+ "eval_multi_news_runtime": 104.4536,
1677
+ "eval_multi_news_samples_per_second": 86.115,
1678
+ "eval_multi_news_steps_per_second": 0.9,
1679
+ "step": 1000
1680
+ },
1681
+ {
1682
+ "epoch": 0.03,
1683
+ "eval_tldr_news_accuracy": 0.5934825543120474,
1684
+ "eval_tldr_news_loss": 1.779296875,
1685
+ "eval_tldr_news_runtime": 7.875,
1686
+ "eval_tldr_news_samples_per_second": 181.334,
1687
+ "eval_tldr_news_steps_per_second": 1.905,
1688
+ "step": 1000
1689
+ },
1690
+ {
1691
+ "epoch": 0.03,
1692
+ "eval_scitldr_accuracy": 0.49756888168557534,
1693
+ "eval_scitldr_loss": NaN,
1694
+ "eval_scitldr_runtime": 5.4836,
1695
+ "eval_scitldr_samples_per_second": 72.763,
1696
+ "eval_scitldr_steps_per_second": 0.912,
1697
+ "step": 1000
1698
+ },
1699
+ {
1700
+ "epoch": 0.03,
1701
+ "eval_samsum_accuracy": 0.6441076667252498,
1702
+ "eval_samsum_loss": 1.3203125,
1703
+ "eval_samsum_runtime": 31.9228,
1704
+ "eval_samsum_samples_per_second": 92.317,
1705
+ "eval_samsum_steps_per_second": 0.971,
1706
+ "step": 1000
1707
+ },
1708
+ {
1709
+ "epoch": 0.03,
1710
+ "eval_debate_sum_accuracy": 0.9393930900916929,
1711
+ "eval_debate_sum_loss": 0.327392578125,
1712
+ "eval_debate_sum_runtime": 546.4353,
1713
+ "eval_debate_sum_samples_per_second": 88.051,
1714
+ "eval_debate_sum_steps_per_second": 0.919,
1715
+ "step": 1000
1716
+ },
1717
+ {
1718
+ "epoch": 0.03,
1719
+ "eval_billsum_accuracy": 0.6859647270039075,
1720
+ "eval_billsum_loss": 1.3212890625,
1721
+ "eval_billsum_runtime": 47.3064,
1722
+ "eval_billsum_samples_per_second": 80.116,
1723
+ "eval_billsum_steps_per_second": 0.846,
1724
+ "step": 1000
1725
+ },
1726
+ {
1727
+ "epoch": 0.03,
1728
+ "eval_wmt2019_zh-en_accuracy": 0.6666222464280335,
1729
+ "eval_wmt2019_zh-en_loss": 1.4609375,
1730
+ "eval_wmt2019_zh-en_runtime": 27.4142,
1731
+ "eval_wmt2019_zh-en_samples_per_second": 145.217,
1732
+ "eval_wmt2019_zh-en_steps_per_second": 1.532,
1733
+ "step": 1000
1734
+ },
1735
+ {
1736
+ "epoch": 0.03,
1737
+ "eval_wmt2019_ru-en_accuracy": 0.7586163428740916,
1738
+ "eval_wmt2019_ru-en_loss": 0.93212890625,
1739
+ "eval_wmt2019_ru-en_runtime": 22.6757,
1740
+ "eval_wmt2019_ru-en_samples_per_second": 132.3,
1741
+ "eval_wmt2019_ru-en_steps_per_second": 1.411,
1742
+ "step": 1000
1743
+ },
1744
+ {
1745
+ "epoch": 0.03,
1746
+ "eval_wmt2019_de-en_accuracy": 0.7644713185146496,
1747
+ "eval_wmt2019_de-en_loss": 0.92724609375,
1748
+ "eval_wmt2019_de-en_runtime": 15.456,
1749
+ "eval_wmt2019_de-en_samples_per_second": 193.97,
1750
+ "eval_wmt2019_de-en_steps_per_second": 2.07,
1751
+ "step": 1000
1752
+ },
1753
+ {
1754
+ "epoch": 0.03,
1755
+ "eval_wmt2019_fr-de_accuracy": 0.7478946231915353,
1756
+ "eval_wmt2019_fr-de_loss": 1.0068359375,
1757
+ "eval_wmt2019_fr-de_runtime": 10.3196,
1758
+ "eval_wmt2019_fr-de_samples_per_second": 146.518,
1759
+ "eval_wmt2019_fr-de_steps_per_second": 1.55,
1760
+ "step": 1000
1761
+ },
1762
+ {
1763
+ "epoch": 0.03,
1764
+ "eval_essay_instruction_accuracy": 0.6048415629215222,
1765
+ "eval_essay_instruction_loss": 1.8955078125,
1766
+ "eval_essay_instruction_runtime": 9.0231,
1767
+ "eval_essay_instruction_samples_per_second": 45.771,
1768
+ "eval_essay_instruction_steps_per_second": 0.554,
1769
+ "step": 1000
1770
+ },
1771
+ {
1772
+ "epoch": 0.03,
1773
+ "eval_reddit_eli5_accuracy": 0.4608785206404607,
1774
+ "eval_reddit_eli5_loss": 2.4296875,
1775
+ "eval_reddit_eli5_runtime": 597.6745,
1776
+ "eval_reddit_eli5_samples_per_second": 91.232,
1777
+ "eval_reddit_eli5_steps_per_second": 0.95,
1778
+ "step": 1000
1779
+ },
1780
+ {
1781
+ "epoch": 0.03,
1782
+ "eval_reddit_askh_accuracy": 0.4638486660338061,
1783
+ "eval_reddit_askh_loss": 2.5234375,
1784
+ "eval_reddit_askh_runtime": 248.9187,
1785
+ "eval_reddit_askh_samples_per_second": 79.162,
1786
+ "eval_reddit_askh_steps_per_second": 0.828,
1787
+ "step": 1000
1788
+ },
1789
+ {
1790
+ "epoch": 0.03,
1791
+ "eval_reddit_asks_accuracy": 0.4715865474658219,
1792
+ "eval_reddit_asks_loss": 2.384765625,
1793
+ "eval_reddit_asks_runtime": 310.7333,
1794
+ "eval_reddit_asks_samples_per_second": 84.819,
1795
+ "eval_reddit_asks_steps_per_second": 0.885,
1796
+ "step": 1000
1797
+ }
1798
+ ],
1799
+ "max_steps": 67822,
1800
+ "num_train_epochs": 2,
1801
+ "total_flos": 1.7293861155088892e+19,
1802
+ "trial_name": null,
1803
+ "trial_params": null
1804
+ }