yiran-wang3 commited on
Commit
c18b83e
1 Parent(s): 02f0b59

End of training

Browse files
README.md ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ license: other
4
+ base_model: yiran-wang3/ds_coder_adamw_iter3
5
+ tags:
6
+ - alignment-handbook
7
+ - generated_from_trainer
8
+ - trl
9
+ - dpo
10
+ datasets:
11
+ - self-generate/ds_coder_sppo_hard_new_cn_mining_oj_iter3-binarized
12
+ model-index:
13
+ - name: ds_coder_adamw_iter4
14
+ results: []
15
+ ---
16
+
17
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
18
+ should probably proofread and complete it, then remove this comment. -->
19
+
20
+ # ds_coder_adamw_iter4
21
+
22
+ This model is a fine-tuned version of [yiran-wang3/ds_coder_adamw_iter3](https://huggingface.co/yiran-wang3/ds_coder_adamw_iter3) on the self-generate/ds_coder_sppo_hard_new_cn_mining_oj_iter3-binarized dataset.
23
+
24
+ ## Model description
25
+
26
+ More information needed
27
+
28
+ ## Intended uses & limitations
29
+
30
+ More information needed
31
+
32
+ ## Training and evaluation data
33
+
34
+ More information needed
35
+
36
+ ## Training procedure
37
+
38
+ ### Training hyperparameters
39
+
40
+ The following hyperparameters were used during training:
41
+ - learning_rate: 1e-06
42
+ - train_batch_size: 8
43
+ - eval_batch_size: 4
44
+ - seed: 42
45
+ - distributed_type: multi-GPU
46
+ - num_devices: 8
47
+ - total_train_batch_size: 64
48
+ - total_eval_batch_size: 32
49
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
50
+ - lr_scheduler_type: constant
51
+ - lr_scheduler_warmup_ratio: 0.1
52
+ - lr_scheduler_warmup_steps: 100
53
+ - num_epochs: 1.0
54
+
55
+ ### Training results
56
+
57
+
58
+
59
+ ### Framework versions
60
+
61
+ - Transformers 4.45.0
62
+ - Pytorch 2.4.0+cu121
63
+ - Datasets 2.14.6
64
+ - Tokenizers 0.20.3
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "total_flos": 0.0,
4
+ "train_loss": 0.4969010160249822,
5
+ "train_runtime": 407.6559,
6
+ "train_samples": 2139,
7
+ "train_samples_per_second": 5.247,
8
+ "train_steps_per_second": 0.083
9
+ }
config.json CHANGED
@@ -25,6 +25,6 @@
25
  "tie_word_embeddings": false,
26
  "torch_dtype": "bfloat16",
27
  "transformers_version": "4.45.0",
28
- "use_cache": false,
29
  "vocab_size": 102400
30
  }
 
25
  "tie_word_embeddings": false,
26
  "torch_dtype": "bfloat16",
27
  "transformers_version": "4.45.0",
28
+ "use_cache": true,
29
  "vocab_size": 102400
30
  }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 100000,
4
+ "eos_token_id": 100015,
5
+ "transformers_version": "4.45.0"
6
+ }
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "total_flos": 0.0,
4
+ "train_loss": 0.4969010160249822,
5
+ "train_runtime": 407.6559,
6
+ "train_samples": 2139,
7
+ "train_samples_per_second": 5.247,
8
+ "train_steps_per_second": 0.083
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,756 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
+ "eval_steps": 100,
6
+ "global_step": 34,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "debug/policy_chosen_logits": 29.054555892944336,
13
+ "debug/policy_chosen_logps": -394.62103271484375,
14
+ "debug/policy_rejected_logits": 30.527902603149414,
15
+ "debug/policy_rejected_logps": -401.28692626953125,
16
+ "debug/reference_chosen_logps": -394.62103271484375,
17
+ "debug/reference_rejected_logps": -401.28692626953125,
18
+ "epoch": 0.029411764705882353,
19
+ "grad_norm": 5.461222893396727,
20
+ "learning_rate": 1e-06,
21
+ "logits/chosen": 29.054555892944336,
22
+ "logits/rejected": 30.527902603149414,
23
+ "logps/chosen": -394.62103271484375,
24
+ "logps/rejected": -401.28692626953125,
25
+ "loss": 0.5,
26
+ "rewards/accuracies": 0.0,
27
+ "rewards/chosen": 0.0,
28
+ "rewards/margins": 0.0,
29
+ "rewards/rejected": 0.0,
30
+ "step": 1
31
+ },
32
+ {
33
+ "debug/policy_chosen_logits": 26.592811584472656,
34
+ "debug/policy_chosen_logps": -404.9052429199219,
35
+ "debug/policy_rejected_logits": 23.82817268371582,
36
+ "debug/policy_rejected_logps": -377.16888427734375,
37
+ "debug/reference_chosen_logps": -405.39239501953125,
38
+ "debug/reference_rejected_logps": -377.3977966308594,
39
+ "epoch": 0.058823529411764705,
40
+ "grad_norm": 5.26155447031347,
41
+ "learning_rate": 1e-06,
42
+ "logits/chosen": 26.592811584472656,
43
+ "logits/rejected": 23.82817268371582,
44
+ "logps/chosen": -404.9052429199219,
45
+ "logps/rejected": -377.16888427734375,
46
+ "loss": 0.4996,
47
+ "rewards/accuracies": 0.625,
48
+ "rewards/chosen": 0.004871406592428684,
49
+ "rewards/margins": 0.002582397311925888,
50
+ "rewards/rejected": 0.002289008814841509,
51
+ "step": 2
52
+ },
53
+ {
54
+ "debug/policy_chosen_logits": 29.616479873657227,
55
+ "debug/policy_chosen_logps": -402.9261474609375,
56
+ "debug/policy_rejected_logits": 27.229713439941406,
57
+ "debug/policy_rejected_logps": -402.63165283203125,
58
+ "debug/reference_chosen_logps": -402.37353515625,
59
+ "debug/reference_rejected_logps": -403.0528564453125,
60
+ "epoch": 0.08823529411764706,
61
+ "grad_norm": 5.686734369927699,
62
+ "learning_rate": 1e-06,
63
+ "logits/chosen": 29.616479873657227,
64
+ "logits/rejected": 27.229713439941406,
65
+ "logps/chosen": -402.9261474609375,
66
+ "logps/rejected": -402.63165283203125,
67
+ "loss": 0.5037,
68
+ "rewards/accuracies": 0.375,
69
+ "rewards/chosen": -0.005526161286979914,
70
+ "rewards/margins": -0.009738006629049778,
71
+ "rewards/rejected": 0.004211845342069864,
72
+ "step": 3
73
+ },
74
+ {
75
+ "debug/policy_chosen_logits": 28.472673416137695,
76
+ "debug/policy_chosen_logps": -406.1566467285156,
77
+ "debug/policy_rejected_logits": 28.92209815979004,
78
+ "debug/policy_rejected_logps": -408.373046875,
79
+ "debug/reference_chosen_logps": -406.66375732421875,
80
+ "debug/reference_rejected_logps": -407.8741760253906,
81
+ "epoch": 0.11764705882352941,
82
+ "grad_norm": 5.378959566289513,
83
+ "learning_rate": 1e-06,
84
+ "logits/chosen": 28.472673416137695,
85
+ "logits/rejected": 28.92209815979004,
86
+ "logps/chosen": -406.1566467285156,
87
+ "logps/rejected": -408.373046875,
88
+ "loss": 0.5029,
89
+ "rewards/accuracies": 0.5,
90
+ "rewards/chosen": 0.005071220453828573,
91
+ "rewards/margins": 0.010060157626867294,
92
+ "rewards/rejected": -0.004988937638700008,
93
+ "step": 4
94
+ },
95
+ {
96
+ "debug/policy_chosen_logits": 31.492525100708008,
97
+ "debug/policy_chosen_logps": -403.90069580078125,
98
+ "debug/policy_rejected_logits": 30.92120361328125,
99
+ "debug/policy_rejected_logps": -411.68096923828125,
100
+ "debug/reference_chosen_logps": -404.43756103515625,
101
+ "debug/reference_rejected_logps": -412.2107849121094,
102
+ "epoch": 0.14705882352941177,
103
+ "grad_norm": 5.270120936652155,
104
+ "learning_rate": 1e-06,
105
+ "logits/chosen": 31.492525100708008,
106
+ "logits/rejected": 30.92120361328125,
107
+ "logps/chosen": -403.90069580078125,
108
+ "logps/rejected": -411.68096923828125,
109
+ "loss": 0.4989,
110
+ "rewards/accuracies": 0.5,
111
+ "rewards/chosen": 0.005368461366742849,
112
+ "rewards/margins": 7.053330773487687e-05,
113
+ "rewards/rejected": 0.005297927651554346,
114
+ "step": 5
115
+ },
116
+ {
117
+ "debug/policy_chosen_logits": 31.689584732055664,
118
+ "debug/policy_chosen_logps": -426.2255554199219,
119
+ "debug/policy_rejected_logits": 31.215805053710938,
120
+ "debug/policy_rejected_logps": -414.5820007324219,
121
+ "debug/reference_chosen_logps": -426.4078369140625,
122
+ "debug/reference_rejected_logps": -414.718017578125,
123
+ "epoch": 0.17647058823529413,
124
+ "grad_norm": 5.511863687507664,
125
+ "learning_rate": 1e-06,
126
+ "logits/chosen": 31.689584732055664,
127
+ "logits/rejected": 31.215805053710938,
128
+ "logps/chosen": -426.2255554199219,
129
+ "logps/rejected": -414.5820007324219,
130
+ "loss": 0.5044,
131
+ "rewards/accuracies": 0.5,
132
+ "rewards/chosen": 0.001822624122723937,
133
+ "rewards/margins": 0.0004624177236109972,
134
+ "rewards/rejected": 0.0013602066319435835,
135
+ "step": 6
136
+ },
137
+ {
138
+ "debug/policy_chosen_logits": 27.33298683166504,
139
+ "debug/policy_chosen_logps": -410.829345703125,
140
+ "debug/policy_rejected_logits": 26.893630981445312,
141
+ "debug/policy_rejected_logps": -394.69989013671875,
142
+ "debug/reference_chosen_logps": -410.4607849121094,
143
+ "debug/reference_rejected_logps": -395.7179870605469,
144
+ "epoch": 0.20588235294117646,
145
+ "grad_norm": 5.290456967833764,
146
+ "learning_rate": 1e-06,
147
+ "logits/chosen": 27.33298683166504,
148
+ "logits/rejected": 26.893630981445312,
149
+ "logps/chosen": -410.829345703125,
150
+ "logps/rejected": -394.69989013671875,
151
+ "loss": 0.501,
152
+ "rewards/accuracies": 0.25,
153
+ "rewards/chosen": -0.003685684408992529,
154
+ "rewards/margins": -0.013866348192095757,
155
+ "rewards/rejected": 0.010180664248764515,
156
+ "step": 7
157
+ },
158
+ {
159
+ "debug/policy_chosen_logits": 26.145978927612305,
160
+ "debug/policy_chosen_logps": -409.4605712890625,
161
+ "debug/policy_rejected_logits": 27.502052307128906,
162
+ "debug/policy_rejected_logps": -401.6300048828125,
163
+ "debug/reference_chosen_logps": -409.36480712890625,
164
+ "debug/reference_rejected_logps": -402.3456726074219,
165
+ "epoch": 0.23529411764705882,
166
+ "grad_norm": 5.259206650868919,
167
+ "learning_rate": 1e-06,
168
+ "logits/chosen": 26.145978927612305,
169
+ "logits/rejected": 27.502052307128906,
170
+ "logps/chosen": -409.4605712890625,
171
+ "logps/rejected": -401.6300048828125,
172
+ "loss": 0.4965,
173
+ "rewards/accuracies": 0.375,
174
+ "rewards/chosen": -0.0009576798183843493,
175
+ "rewards/margins": -0.00811409018933773,
176
+ "rewards/rejected": 0.007156409788876772,
177
+ "step": 8
178
+ },
179
+ {
180
+ "debug/policy_chosen_logits": 29.184545516967773,
181
+ "debug/policy_chosen_logps": -393.89373779296875,
182
+ "debug/policy_rejected_logits": 28.960050582885742,
183
+ "debug/policy_rejected_logps": -382.4853820800781,
184
+ "debug/reference_chosen_logps": -393.92431640625,
185
+ "debug/reference_rejected_logps": -382.33984375,
186
+ "epoch": 0.2647058823529412,
187
+ "grad_norm": 5.662262990371031,
188
+ "learning_rate": 1e-06,
189
+ "logits/chosen": 29.184545516967773,
190
+ "logits/rejected": 28.960050582885742,
191
+ "logps/chosen": -393.89373779296875,
192
+ "logps/rejected": -382.4853820800781,
193
+ "loss": 0.5007,
194
+ "rewards/accuracies": 0.5,
195
+ "rewards/chosen": 0.0003057096619158983,
196
+ "rewards/margins": 0.001760864513926208,
197
+ "rewards/rejected": -0.0014551542699337006,
198
+ "step": 9
199
+ },
200
+ {
201
+ "debug/policy_chosen_logits": 24.076282501220703,
202
+ "debug/policy_chosen_logps": -394.2970275878906,
203
+ "debug/policy_rejected_logits": 21.97032928466797,
204
+ "debug/policy_rejected_logps": -381.5108947753906,
205
+ "debug/reference_chosen_logps": -393.9175720214844,
206
+ "debug/reference_rejected_logps": -381.532470703125,
207
+ "epoch": 0.29411764705882354,
208
+ "grad_norm": 5.055283348007549,
209
+ "learning_rate": 1e-06,
210
+ "logits/chosen": 24.076282501220703,
211
+ "logits/rejected": 21.97032928466797,
212
+ "logps/chosen": -394.2970275878906,
213
+ "logps/rejected": -381.5108947753906,
214
+ "loss": 0.4978,
215
+ "rewards/accuracies": 0.5,
216
+ "rewards/chosen": -0.0037943648640066385,
217
+ "rewards/margins": -0.004009971860796213,
218
+ "rewards/rejected": 0.0002156068803742528,
219
+ "step": 10
220
+ },
221
+ {
222
+ "debug/policy_chosen_logits": 30.054981231689453,
223
+ "debug/policy_chosen_logps": -386.2349853515625,
224
+ "debug/policy_rejected_logits": 29.39519691467285,
225
+ "debug/policy_rejected_logps": -412.88177490234375,
226
+ "debug/reference_chosen_logps": -386.55413818359375,
227
+ "debug/reference_rejected_logps": -413.00701904296875,
228
+ "epoch": 0.3235294117647059,
229
+ "grad_norm": 5.392953865442379,
230
+ "learning_rate": 1e-06,
231
+ "logits/chosen": 30.054981231689453,
232
+ "logits/rejected": 29.39519691467285,
233
+ "logps/chosen": -386.2349853515625,
234
+ "logps/rejected": -412.88177490234375,
235
+ "loss": 0.4972,
236
+ "rewards/accuracies": 0.75,
237
+ "rewards/chosen": 0.0031916426960378885,
238
+ "rewards/margins": 0.0019394302507862449,
239
+ "rewards/rejected": 0.0012522124452516437,
240
+ "step": 11
241
+ },
242
+ {
243
+ "debug/policy_chosen_logits": 27.566011428833008,
244
+ "debug/policy_chosen_logps": -374.5032653808594,
245
+ "debug/policy_rejected_logits": 29.68182373046875,
246
+ "debug/policy_rejected_logps": -401.15179443359375,
247
+ "debug/reference_chosen_logps": -374.087646484375,
248
+ "debug/reference_rejected_logps": -402.13214111328125,
249
+ "epoch": 0.35294117647058826,
250
+ "grad_norm": 6.451284347500255,
251
+ "learning_rate": 1e-06,
252
+ "logits/chosen": 27.566011428833008,
253
+ "logits/rejected": 29.68182373046875,
254
+ "logps/chosen": -374.5032653808594,
255
+ "logps/rejected": -401.15179443359375,
256
+ "loss": 0.4998,
257
+ "rewards/accuracies": 0.25,
258
+ "rewards/chosen": -0.0041561126708984375,
259
+ "rewards/margins": -0.013959312811493874,
260
+ "rewards/rejected": 0.009803200140595436,
261
+ "step": 12
262
+ },
263
+ {
264
+ "debug/policy_chosen_logits": 28.40235137939453,
265
+ "debug/policy_chosen_logps": -399.9848327636719,
266
+ "debug/policy_rejected_logits": 29.182313919067383,
267
+ "debug/policy_rejected_logps": -415.01519775390625,
268
+ "debug/reference_chosen_logps": -400.4940185546875,
269
+ "debug/reference_rejected_logps": -414.45550537109375,
270
+ "epoch": 0.38235294117647056,
271
+ "grad_norm": 4.8489563556671005,
272
+ "learning_rate": 1e-06,
273
+ "logits/chosen": 28.40235137939453,
274
+ "logits/rejected": 29.182313919067383,
275
+ "logps/chosen": -399.9848327636719,
276
+ "logps/rejected": -415.01519775390625,
277
+ "loss": 0.4979,
278
+ "rewards/accuracies": 0.75,
279
+ "rewards/chosen": 0.005092009902000427,
280
+ "rewards/margins": 0.010688590817153454,
281
+ "rewards/rejected": -0.005596580449491739,
282
+ "step": 13
283
+ },
284
+ {
285
+ "debug/policy_chosen_logits": 21.881561279296875,
286
+ "debug/policy_chosen_logps": -402.83416748046875,
287
+ "debug/policy_rejected_logits": 23.709701538085938,
288
+ "debug/policy_rejected_logps": -417.4763488769531,
289
+ "debug/reference_chosen_logps": -402.1183166503906,
290
+ "debug/reference_rejected_logps": -417.23797607421875,
291
+ "epoch": 0.4117647058823529,
292
+ "grad_norm": 5.498578228190857,
293
+ "learning_rate": 1e-06,
294
+ "logits/chosen": 21.881561279296875,
295
+ "logits/rejected": 23.709701538085938,
296
+ "logps/chosen": -402.83416748046875,
297
+ "logps/rejected": -417.4763488769531,
298
+ "loss": 0.5019,
299
+ "rewards/accuracies": 0.625,
300
+ "rewards/chosen": -0.007158737629652023,
301
+ "rewards/margins": -0.004775161389261484,
302
+ "rewards/rejected": -0.0023835753090679646,
303
+ "step": 14
304
+ },
305
+ {
306
+ "debug/policy_chosen_logits": 29.381258010864258,
307
+ "debug/policy_chosen_logps": -395.54803466796875,
308
+ "debug/policy_rejected_logits": 28.470420837402344,
309
+ "debug/policy_rejected_logps": -387.3695373535156,
310
+ "debug/reference_chosen_logps": -395.508056640625,
311
+ "debug/reference_rejected_logps": -386.67974853515625,
312
+ "epoch": 0.4411764705882353,
313
+ "grad_norm": 6.635697402918325,
314
+ "learning_rate": 1e-06,
315
+ "logits/chosen": 29.381258010864258,
316
+ "logits/rejected": 28.470420837402344,
317
+ "logps/chosen": -395.54803466796875,
318
+ "logps/rejected": -387.3695373535156,
319
+ "loss": 0.4963,
320
+ "rewards/accuracies": 0.5,
321
+ "rewards/chosen": -0.0003996272571384907,
322
+ "rewards/margins": 0.006498145870864391,
323
+ "rewards/rejected": -0.006897774059325457,
324
+ "step": 15
325
+ },
326
+ {
327
+ "debug/policy_chosen_logits": 27.256446838378906,
328
+ "debug/policy_chosen_logps": -403.69305419921875,
329
+ "debug/policy_rejected_logits": 26.9769344329834,
330
+ "debug/policy_rejected_logps": -387.86083984375,
331
+ "debug/reference_chosen_logps": -404.33697509765625,
332
+ "debug/reference_rejected_logps": -388.44683837890625,
333
+ "epoch": 0.47058823529411764,
334
+ "grad_norm": 5.539928310976471,
335
+ "learning_rate": 1e-06,
336
+ "logits/chosen": 27.256446838378906,
337
+ "logits/rejected": 26.9769344329834,
338
+ "logps/chosen": -403.69305419921875,
339
+ "logps/rejected": -387.86083984375,
340
+ "loss": 0.497,
341
+ "rewards/accuracies": 0.5,
342
+ "rewards/chosen": 0.006439666263759136,
343
+ "rewards/margins": 0.0005795289762318134,
344
+ "rewards/rejected": 0.00586013775318861,
345
+ "step": 16
346
+ },
347
+ {
348
+ "debug/policy_chosen_logits": 27.54875373840332,
349
+ "debug/policy_chosen_logps": -379.23291015625,
350
+ "debug/policy_rejected_logits": 28.37594985961914,
351
+ "debug/policy_rejected_logps": -391.45654296875,
352
+ "debug/reference_chosen_logps": -380.18060302734375,
353
+ "debug/reference_rejected_logps": -390.98968505859375,
354
+ "epoch": 0.5,
355
+ "grad_norm": 5.2957197578010415,
356
+ "learning_rate": 1e-06,
357
+ "logits/chosen": 27.54875373840332,
358
+ "logits/rejected": 28.37594985961914,
359
+ "logps/chosen": -379.23291015625,
360
+ "logps/rejected": -391.45654296875,
361
+ "loss": 0.4938,
362
+ "rewards/accuracies": 0.75,
363
+ "rewards/chosen": 0.009477118961513042,
364
+ "rewards/margins": 0.01414569839835167,
365
+ "rewards/rejected": -0.0046685789711773396,
366
+ "step": 17
367
+ },
368
+ {
369
+ "debug/policy_chosen_logits": 25.160919189453125,
370
+ "debug/policy_chosen_logps": -389.9479675292969,
371
+ "debug/policy_rejected_logits": 25.0775146484375,
372
+ "debug/policy_rejected_logps": -372.14349365234375,
373
+ "debug/reference_chosen_logps": -389.97222900390625,
374
+ "debug/reference_rejected_logps": -371.96075439453125,
375
+ "epoch": 0.5294117647058824,
376
+ "grad_norm": 4.954808294891358,
377
+ "learning_rate": 1e-06,
378
+ "logits/chosen": 25.160919189453125,
379
+ "logits/rejected": 25.0775146484375,
380
+ "logps/chosen": -389.9479675292969,
381
+ "logps/rejected": -372.14349365234375,
382
+ "loss": 0.4953,
383
+ "rewards/accuracies": 0.375,
384
+ "rewards/chosen": 0.00024272920563817024,
385
+ "rewards/margins": 0.002069931011646986,
386
+ "rewards/rejected": -0.0018272020388394594,
387
+ "step": 18
388
+ },
389
+ {
390
+ "debug/policy_chosen_logits": 25.586524963378906,
391
+ "debug/policy_chosen_logps": -374.25006103515625,
392
+ "debug/policy_rejected_logits": 28.893770217895508,
393
+ "debug/policy_rejected_logps": -412.3420104980469,
394
+ "debug/reference_chosen_logps": -374.282958984375,
395
+ "debug/reference_rejected_logps": -411.41131591796875,
396
+ "epoch": 0.5588235294117647,
397
+ "grad_norm": 4.9850918433013724,
398
+ "learning_rate": 1e-06,
399
+ "logits/chosen": 25.586524963378906,
400
+ "logits/rejected": 28.893770217895508,
401
+ "logps/chosen": -374.25006103515625,
402
+ "logps/rejected": -412.3420104980469,
403
+ "loss": 0.5002,
404
+ "rewards/accuracies": 0.375,
405
+ "rewards/chosen": 0.0003293610643595457,
406
+ "rewards/margins": 0.009636610746383667,
407
+ "rewards/rejected": -0.009307250380516052,
408
+ "step": 19
409
+ },
410
+ {
411
+ "debug/policy_chosen_logits": 24.718896865844727,
412
+ "debug/policy_chosen_logps": -363.05120849609375,
413
+ "debug/policy_rejected_logits": 24.986190795898438,
414
+ "debug/policy_rejected_logps": -388.0316162109375,
415
+ "debug/reference_chosen_logps": -362.204833984375,
416
+ "debug/reference_rejected_logps": -386.9569091796875,
417
+ "epoch": 0.5882352941176471,
418
+ "grad_norm": 5.05147280128332,
419
+ "learning_rate": 1e-06,
420
+ "logits/chosen": 24.718896865844727,
421
+ "logits/rejected": 24.986190795898438,
422
+ "logps/chosen": -363.05120849609375,
423
+ "logps/rejected": -388.0316162109375,
424
+ "loss": 0.4997,
425
+ "rewards/accuracies": 0.5,
426
+ "rewards/chosen": -0.00846382137387991,
427
+ "rewards/margins": 0.0022830581292510033,
428
+ "rewards/rejected": -0.010746879503130913,
429
+ "step": 20
430
+ },
431
+ {
432
+ "debug/policy_chosen_logits": 28.18993377685547,
433
+ "debug/policy_chosen_logps": -397.7740478515625,
434
+ "debug/policy_rejected_logits": 23.599760055541992,
435
+ "debug/policy_rejected_logps": -400.73492431640625,
436
+ "debug/reference_chosen_logps": -399.1761169433594,
437
+ "debug/reference_rejected_logps": -399.78948974609375,
438
+ "epoch": 0.6176470588235294,
439
+ "grad_norm": 6.701576850276441,
440
+ "learning_rate": 1e-06,
441
+ "logits/chosen": 28.18993377685547,
442
+ "logits/rejected": 23.599760055541992,
443
+ "logps/chosen": -397.7740478515625,
444
+ "logps/rejected": -400.73492431640625,
445
+ "loss": 0.4981,
446
+ "rewards/accuracies": 0.5,
447
+ "rewards/chosen": 0.014020653441548347,
448
+ "rewards/margins": 0.023475077003240585,
449
+ "rewards/rejected": -0.009454423561692238,
450
+ "step": 21
451
+ },
452
+ {
453
+ "debug/policy_chosen_logits": 29.8790283203125,
454
+ "debug/policy_chosen_logps": -413.7144775390625,
455
+ "debug/policy_rejected_logits": 28.82781982421875,
456
+ "debug/policy_rejected_logps": -409.18292236328125,
457
+ "debug/reference_chosen_logps": -414.69781494140625,
458
+ "debug/reference_rejected_logps": -407.171142578125,
459
+ "epoch": 0.6470588235294118,
460
+ "grad_norm": 6.323947957916722,
461
+ "learning_rate": 1e-06,
462
+ "logits/chosen": 29.8790283203125,
463
+ "logits/rejected": 28.82781982421875,
464
+ "logps/chosen": -413.7144775390625,
465
+ "logps/rejected": -409.18292236328125,
466
+ "loss": 0.4999,
467
+ "rewards/accuracies": 0.5,
468
+ "rewards/chosen": 0.009833297692239285,
469
+ "rewards/margins": 0.029951322823762894,
470
+ "rewards/rejected": -0.020118026062846184,
471
+ "step": 22
472
+ },
473
+ {
474
+ "debug/policy_chosen_logits": 27.423133850097656,
475
+ "debug/policy_chosen_logps": -377.9825439453125,
476
+ "debug/policy_rejected_logits": 29.817766189575195,
477
+ "debug/policy_rejected_logps": -400.9486389160156,
478
+ "debug/reference_chosen_logps": -378.6519775390625,
479
+ "debug/reference_rejected_logps": -400.3038024902344,
480
+ "epoch": 0.6764705882352942,
481
+ "grad_norm": 7.650698869625993,
482
+ "learning_rate": 1e-06,
483
+ "logits/chosen": 27.423133850097656,
484
+ "logits/rejected": 29.817766189575195,
485
+ "logps/chosen": -377.9825439453125,
486
+ "logps/rejected": -400.9486389160156,
487
+ "loss": 0.4944,
488
+ "rewards/accuracies": 0.875,
489
+ "rewards/chosen": 0.006694145500659943,
490
+ "rewards/margins": 0.013142585754394531,
491
+ "rewards/rejected": -0.006448440253734589,
492
+ "step": 23
493
+ },
494
+ {
495
+ "debug/policy_chosen_logits": 26.779478073120117,
496
+ "debug/policy_chosen_logps": -392.8223571777344,
497
+ "debug/policy_rejected_logits": 28.28175163269043,
498
+ "debug/policy_rejected_logps": -394.74676513671875,
499
+ "debug/reference_chosen_logps": -393.3936462402344,
500
+ "debug/reference_rejected_logps": -394.11358642578125,
501
+ "epoch": 0.7058823529411765,
502
+ "grad_norm": 5.251742483181524,
503
+ "learning_rate": 1e-06,
504
+ "logits/chosen": 26.779478073120117,
505
+ "logits/rejected": 28.28175163269043,
506
+ "logps/chosen": -392.8223571777344,
507
+ "logps/rejected": -394.74676513671875,
508
+ "loss": 0.5016,
509
+ "rewards/accuracies": 0.5,
510
+ "rewards/chosen": 0.005712966434657574,
511
+ "rewards/margins": 0.012044791132211685,
512
+ "rewards/rejected": -0.0063318246975541115,
513
+ "step": 24
514
+ },
515
+ {
516
+ "debug/policy_chosen_logits": 28.304908752441406,
517
+ "debug/policy_chosen_logps": -395.7744445800781,
518
+ "debug/policy_rejected_logits": 27.35923957824707,
519
+ "debug/policy_rejected_logps": -403.1441955566406,
520
+ "debug/reference_chosen_logps": -395.40789794921875,
521
+ "debug/reference_rejected_logps": -403.5139465332031,
522
+ "epoch": 0.7352941176470589,
523
+ "grad_norm": 4.968117146678153,
524
+ "learning_rate": 1e-06,
525
+ "logits/chosen": 28.304908752441406,
526
+ "logits/rejected": 27.35923957824707,
527
+ "logps/chosen": -395.7744445800781,
528
+ "logps/rejected": -403.1441955566406,
529
+ "loss": 0.4985,
530
+ "rewards/accuracies": 0.125,
531
+ "rewards/chosen": -0.003665314055979252,
532
+ "rewards/margins": -0.007362784817814827,
533
+ "rewards/rejected": 0.0036974712274968624,
534
+ "step": 25
535
+ },
536
+ {
537
+ "debug/policy_chosen_logits": 28.73921012878418,
538
+ "debug/policy_chosen_logps": -396.11834716796875,
539
+ "debug/policy_rejected_logits": 28.473979949951172,
540
+ "debug/policy_rejected_logps": -419.5787353515625,
541
+ "debug/reference_chosen_logps": -396.2462158203125,
542
+ "debug/reference_rejected_logps": -418.2436218261719,
543
+ "epoch": 0.7647058823529411,
544
+ "grad_norm": 5.07020088847226,
545
+ "learning_rate": 1e-06,
546
+ "logits/chosen": 28.73921012878418,
547
+ "logits/rejected": 28.473979949951172,
548
+ "logps/chosen": -396.11834716796875,
549
+ "logps/rejected": -419.5787353515625,
550
+ "loss": 0.4994,
551
+ "rewards/accuracies": 0.75,
552
+ "rewards/chosen": 0.0012784956488758326,
553
+ "rewards/margins": 0.014629820361733437,
554
+ "rewards/rejected": -0.013351325877010822,
555
+ "step": 26
556
+ },
557
+ {
558
+ "debug/policy_chosen_logits": 30.456932067871094,
559
+ "debug/policy_chosen_logps": -410.0908203125,
560
+ "debug/policy_rejected_logits": 28.822614669799805,
561
+ "debug/policy_rejected_logps": -418.9249572753906,
562
+ "debug/reference_chosen_logps": -410.21063232421875,
563
+ "debug/reference_rejected_logps": -417.00714111328125,
564
+ "epoch": 0.7941176470588235,
565
+ "grad_norm": 6.10176209535134,
566
+ "learning_rate": 1e-06,
567
+ "logits/chosen": 30.456932067871094,
568
+ "logits/rejected": 28.822614669799805,
569
+ "logps/chosen": -410.0908203125,
570
+ "logps/rejected": -418.9249572753906,
571
+ "loss": 0.4853,
572
+ "rewards/accuracies": 0.625,
573
+ "rewards/chosen": 0.0011978147085756063,
574
+ "rewards/margins": 0.020375745370984077,
575
+ "rewards/rejected": -0.019177932292222977,
576
+ "step": 27
577
+ },
578
+ {
579
+ "debug/policy_chosen_logits": 28.178850173950195,
580
+ "debug/policy_chosen_logps": -408.07464599609375,
581
+ "debug/policy_rejected_logits": 28.366477966308594,
582
+ "debug/policy_rejected_logps": -414.43853759765625,
583
+ "debug/reference_chosen_logps": -407.51043701171875,
584
+ "debug/reference_rejected_logps": -410.44268798828125,
585
+ "epoch": 0.8235294117647058,
586
+ "grad_norm": 4.919999726189566,
587
+ "learning_rate": 1e-06,
588
+ "logits/chosen": 28.178850173950195,
589
+ "logits/rejected": 28.366477966308594,
590
+ "logps/chosen": -408.07464599609375,
591
+ "logps/rejected": -414.43853759765625,
592
+ "loss": 0.499,
593
+ "rewards/accuracies": 0.625,
594
+ "rewards/chosen": -0.005642051808536053,
595
+ "rewards/margins": 0.03431610018014908,
596
+ "rewards/rejected": -0.039958152920007706,
597
+ "step": 28
598
+ },
599
+ {
600
+ "debug/policy_chosen_logits": 29.156843185424805,
601
+ "debug/policy_chosen_logps": -409.95208740234375,
602
+ "debug/policy_rejected_logits": 27.127243041992188,
603
+ "debug/policy_rejected_logps": -393.4117431640625,
604
+ "debug/reference_chosen_logps": -409.63983154296875,
605
+ "debug/reference_rejected_logps": -392.2528076171875,
606
+ "epoch": 0.8529411764705882,
607
+ "grad_norm": 5.038344787361152,
608
+ "learning_rate": 1e-06,
609
+ "logits/chosen": 29.156843185424805,
610
+ "logits/rejected": 27.127243041992188,
611
+ "logps/chosen": -409.95208740234375,
612
+ "logps/rejected": -393.4117431640625,
613
+ "loss": 0.4935,
614
+ "rewards/accuracies": 0.625,
615
+ "rewards/chosen": -0.003122864291071892,
616
+ "rewards/margins": 0.008466909639537334,
617
+ "rewards/rejected": -0.0115897748619318,
618
+ "step": 29
619
+ },
620
+ {
621
+ "debug/policy_chosen_logits": 26.275718688964844,
622
+ "debug/policy_chosen_logps": -406.03277587890625,
623
+ "debug/policy_rejected_logits": 26.794349670410156,
624
+ "debug/policy_rejected_logps": -410.08282470703125,
625
+ "debug/reference_chosen_logps": -405.848876953125,
626
+ "debug/reference_rejected_logps": -410.5152587890625,
627
+ "epoch": 0.8823529411764706,
628
+ "grad_norm": 6.078237487272278,
629
+ "learning_rate": 1e-06,
630
+ "logits/chosen": 26.275718688964844,
631
+ "logits/rejected": 26.794349670410156,
632
+ "logps/chosen": -406.03277587890625,
633
+ "logps/rejected": -410.08282470703125,
634
+ "loss": 0.4941,
635
+ "rewards/accuracies": 0.5,
636
+ "rewards/chosen": -0.0018388747703284025,
637
+ "rewards/margins": -0.006163253448903561,
638
+ "rewards/rejected": 0.0043243784457445145,
639
+ "step": 30
640
+ },
641
+ {
642
+ "debug/policy_chosen_logits": 27.000301361083984,
643
+ "debug/policy_chosen_logps": -391.47613525390625,
644
+ "debug/policy_rejected_logits": 29.51080322265625,
645
+ "debug/policy_rejected_logps": -386.7709655761719,
646
+ "debug/reference_chosen_logps": -390.591552734375,
647
+ "debug/reference_rejected_logps": -387.83935546875,
648
+ "epoch": 0.9117647058823529,
649
+ "grad_norm": 5.608762984875432,
650
+ "learning_rate": 1e-06,
651
+ "logits/chosen": 27.000301361083984,
652
+ "logits/rejected": 29.51080322265625,
653
+ "logps/chosen": -391.47613525390625,
654
+ "logps/rejected": -386.7709655761719,
655
+ "loss": 0.4983,
656
+ "rewards/accuracies": 0.125,
657
+ "rewards/chosen": -0.008846206590533257,
658
+ "rewards/margins": -0.019530104473233223,
659
+ "rewards/rejected": 0.010683897882699966,
660
+ "step": 31
661
+ },
662
+ {
663
+ "debug/policy_chosen_logits": 29.469999313354492,
664
+ "debug/policy_chosen_logps": -390.48291015625,
665
+ "debug/policy_rejected_logits": 28.674427032470703,
666
+ "debug/policy_rejected_logps": -414.2558288574219,
667
+ "debug/reference_chosen_logps": -392.53704833984375,
668
+ "debug/reference_rejected_logps": -416.4234619140625,
669
+ "epoch": 0.9411764705882353,
670
+ "grad_norm": 5.952413372138852,
671
+ "learning_rate": 1e-06,
672
+ "logits/chosen": 29.469999313354492,
673
+ "logits/rejected": 28.674427032470703,
674
+ "logps/chosen": -390.48291015625,
675
+ "logps/rejected": -414.2558288574219,
676
+ "loss": 0.4928,
677
+ "rewards/accuracies": 0.625,
678
+ "rewards/chosen": 0.02054138109087944,
679
+ "rewards/margins": -0.0011348333209753036,
680
+ "rewards/rejected": 0.021676216274499893,
681
+ "step": 32
682
+ },
683
+ {
684
+ "debug/policy_chosen_logits": 30.11175537109375,
685
+ "debug/policy_chosen_logps": -393.00079345703125,
686
+ "debug/policy_rejected_logits": 31.391122817993164,
687
+ "debug/policy_rejected_logps": -419.4772644042969,
688
+ "debug/reference_chosen_logps": -393.62738037109375,
689
+ "debug/reference_rejected_logps": -416.98974609375,
690
+ "epoch": 0.9705882352941176,
691
+ "grad_norm": 5.204183724206507,
692
+ "learning_rate": 1e-06,
693
+ "logits/chosen": 30.11175537109375,
694
+ "logits/rejected": 31.391122817993164,
695
+ "logps/chosen": -393.00079345703125,
696
+ "logps/rejected": -419.4772644042969,
697
+ "loss": 0.4877,
698
+ "rewards/accuracies": 0.875,
699
+ "rewards/chosen": 0.006265563890337944,
700
+ "rewards/margins": 0.031140554696321487,
701
+ "rewards/rejected": -0.024874988943338394,
702
+ "step": 33
703
+ },
704
+ {
705
+ "debug/policy_chosen_logits": 28.005823135375977,
706
+ "debug/policy_chosen_logps": -417.3971252441406,
707
+ "debug/policy_rejected_logits": 24.94090461730957,
708
+ "debug/policy_rejected_logps": -404.98779296875,
709
+ "debug/reference_chosen_logps": -416.21746826171875,
710
+ "debug/reference_rejected_logps": -404.94940185546875,
711
+ "epoch": 1.0,
712
+ "grad_norm": 5.733599790823146,
713
+ "learning_rate": 1e-06,
714
+ "logits/chosen": 28.005823135375977,
715
+ "logits/rejected": 24.94090461730957,
716
+ "logps/chosen": -417.3971252441406,
717
+ "logps/rejected": -404.98779296875,
718
+ "loss": 0.4673,
719
+ "rewards/accuracies": 0.625,
720
+ "rewards/chosen": -0.011796417646110058,
721
+ "rewards/margins": -0.01141277328133583,
722
+ "rewards/rejected": -0.00038364529609680176,
723
+ "step": 34
724
+ },
725
+ {
726
+ "epoch": 1.0,
727
+ "step": 34,
728
+ "total_flos": 0.0,
729
+ "train_loss": 0.4969010160249822,
730
+ "train_runtime": 407.6559,
731
+ "train_samples_per_second": 5.247,
732
+ "train_steps_per_second": 0.083
733
+ }
734
+ ],
735
+ "logging_steps": 1,
736
+ "max_steps": 34,
737
+ "num_input_tokens_seen": 0,
738
+ "num_train_epochs": 1,
739
+ "save_steps": 500,
740
+ "stateful_callbacks": {
741
+ "TrainerControl": {
742
+ "args": {
743
+ "should_epoch_stop": false,
744
+ "should_evaluate": false,
745
+ "should_log": false,
746
+ "should_save": true,
747
+ "should_training_stop": true
748
+ },
749
+ "attributes": {}
750
+ }
751
+ },
752
+ "total_flos": 0.0,
753
+ "train_batch_size": 8,
754
+ "trial_name": null,
755
+ "trial_params": null
756
+ }