File size: 10,631 Bytes
fbfa663 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9984301412872841,
"eval_steps": 500,
"global_step": 159,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.006279434850863423,
"grad_norm": 17.524188433318294,
"learning_rate": 1.875e-08,
"logits/chosen": 0.050171270966529846,
"logits/rejected": 0.7975481748580933,
"logps/chosen": -213.99826049804688,
"logps/pi_response": -122.33531951904297,
"logps/ref_response": -122.33531951904297,
"logps/rejected": -327.0420227050781,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.06279434850863422,
"grad_norm": 23.45193685304271,
"learning_rate": 1.875e-07,
"logits/chosen": 0.6251156330108643,
"logits/rejected": 0.9096591472625732,
"logps/chosen": -268.11822509765625,
"logps/pi_response": -114.67009735107422,
"logps/ref_response": -114.69570922851562,
"logps/rejected": -390.1163330078125,
"loss": 0.6924,
"rewards/accuracies": 0.5416666865348816,
"rewards/chosen": -0.000606474990490824,
"rewards/margins": 0.0019376208074390888,
"rewards/rejected": -0.0025440959725528955,
"step": 10
},
{
"epoch": 0.12558869701726844,
"grad_norm": 19.551172670207706,
"learning_rate": 2.9942119880575817e-07,
"logits/chosen": 0.4843064248561859,
"logits/rejected": 0.8785603642463684,
"logps/chosen": -269.538330078125,
"logps/pi_response": -121.35018157958984,
"logps/ref_response": -121.34931945800781,
"logps/rejected": -417.247314453125,
"loss": 0.6747,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.0313839390873909,
"rewards/margins": 0.04394400864839554,
"rewards/rejected": -0.07532794773578644,
"step": 20
},
{
"epoch": 0.18838304552590268,
"grad_norm": 13.701658129429845,
"learning_rate": 2.929608750821129e-07,
"logits/chosen": 0.5343824625015259,
"logits/rejected": 0.9908145666122437,
"logps/chosen": -310.638671875,
"logps/pi_response": -118.42143249511719,
"logps/ref_response": -117.36665344238281,
"logps/rejected": -428.20166015625,
"loss": 0.6175,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.18561159074306488,
"rewards/margins": 0.17133654654026031,
"rewards/rejected": -0.3569481372833252,
"step": 30
},
{
"epoch": 0.25117739403453687,
"grad_norm": 14.693007375551543,
"learning_rate": 2.7962832564252725e-07,
"logits/chosen": 0.5080328583717346,
"logits/rejected": 0.9311200976371765,
"logps/chosen": -303.4456481933594,
"logps/pi_response": -121.04774475097656,
"logps/ref_response": -117.7525634765625,
"logps/rejected": -484.09552001953125,
"loss": 0.5785,
"rewards/accuracies": 0.78125,
"rewards/chosen": -0.3230215311050415,
"rewards/margins": 0.4571772515773773,
"rewards/rejected": -0.7801988124847412,
"step": 40
},
{
"epoch": 0.3139717425431711,
"grad_norm": 18.087271424489508,
"learning_rate": 2.6006445513357056e-07,
"logits/chosen": 0.6870445013046265,
"logits/rejected": 0.9769166111946106,
"logps/chosen": -342.7091369628906,
"logps/pi_response": -124.7760009765625,
"logps/ref_response": -115.9255599975586,
"logps/rejected": -495.4518127441406,
"loss": 0.5714,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.712685227394104,
"rewards/margins": 0.5671727657318115,
"rewards/rejected": -1.2798579931259155,
"step": 50
},
{
"epoch": 0.37676609105180536,
"grad_norm": 11.166951031052399,
"learning_rate": 2.3520971200967334e-07,
"logits/chosen": 0.5992540121078491,
"logits/rejected": 1.020711898803711,
"logps/chosen": -348.0439758300781,
"logps/pi_response": -127.2256088256836,
"logps/ref_response": -119.3854751586914,
"logps/rejected": -496.8860778808594,
"loss": 0.5649,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.5891093015670776,
"rewards/margins": 0.5816227793693542,
"rewards/rejected": -1.1707321405410767,
"step": 60
},
{
"epoch": 0.43956043956043955,
"grad_norm": 11.470597805963651,
"learning_rate": 2.0625888054143427e-07,
"logits/chosen": 0.6002563238143921,
"logits/rejected": 0.9772939682006836,
"logps/chosen": -272.6213073730469,
"logps/pi_response": -127.3548583984375,
"logps/ref_response": -120.81624603271484,
"logps/rejected": -524.6642456054688,
"loss": 0.5372,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.36886435747146606,
"rewards/margins": 0.7560935020446777,
"rewards/rejected": -1.124957799911499,
"step": 70
},
{
"epoch": 0.5023547880690737,
"grad_norm": 11.00220562972207,
"learning_rate": 1.7460364672965327e-07,
"logits/chosen": 0.6907710433006287,
"logits/rejected": 1.0923134088516235,
"logps/chosen": -269.1040954589844,
"logps/pi_response": -112.93983459472656,
"logps/ref_response": -106.69105529785156,
"logps/rejected": -492.8182678222656,
"loss": 0.5271,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.4200579524040222,
"rewards/margins": 0.6648958921432495,
"rewards/rejected": -1.084953784942627,
"step": 80
},
{
"epoch": 0.565149136577708,
"grad_norm": 11.111343223576297,
"learning_rate": 1.4176569902035086e-07,
"logits/chosen": 0.6897394061088562,
"logits/rejected": 1.039529800415039,
"logps/chosen": -341.63275146484375,
"logps/pi_response": -120.40811920166016,
"logps/ref_response": -111.32686614990234,
"logps/rejected": -511.60565185546875,
"loss": 0.5055,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.5421563982963562,
"rewards/margins": 0.7095355987548828,
"rewards/rejected": -1.2516921758651733,
"step": 90
},
{
"epoch": 0.6279434850863422,
"grad_norm": 12.249034688194289,
"learning_rate": 1.0932357971453743e-07,
"logits/chosen": 0.7736852765083313,
"logits/rejected": 1.0484180450439453,
"logps/chosen": -290.811279296875,
"logps/pi_response": -116.5997543334961,
"logps/ref_response": -107.03324127197266,
"logps/rejected": -533.04931640625,
"loss": 0.539,
"rewards/accuracies": 0.7562500238418579,
"rewards/chosen": -0.49749264121055603,
"rewards/margins": 0.8334757089614868,
"rewards/rejected": -1.3309683799743652,
"step": 100
},
{
"epoch": 0.6907378335949764,
"grad_norm": 10.864663994027794,
"learning_rate": 7.883680337481599e-08,
"logits/chosen": 0.7670079469680786,
"logits/rejected": 0.9956636428833008,
"logps/chosen": -305.14593505859375,
"logps/pi_response": -127.0176773071289,
"logps/ref_response": -116.74520111083984,
"logps/rejected": -521.2225341796875,
"loss": 0.5335,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.5058807134628296,
"rewards/margins": 0.7484409809112549,
"rewards/rejected": -1.2543216943740845,
"step": 110
},
{
"epoch": 0.7535321821036107,
"grad_norm": 11.061608119367962,
"learning_rate": 5.177088990820725e-08,
"logits/chosen": 0.5085455775260925,
"logits/rejected": 0.8686326146125793,
"logps/chosen": -308.0654602050781,
"logps/pi_response": -132.87335205078125,
"logps/ref_response": -123.3708267211914,
"logps/rejected": -556.9476928710938,
"loss": 0.5259,
"rewards/accuracies": 0.7562500238418579,
"rewards/chosen": -0.4727388918399811,
"rewards/margins": 0.8706821203231812,
"rewards/rejected": -1.3434208631515503,
"step": 120
},
{
"epoch": 0.8163265306122449,
"grad_norm": 13.028590054698721,
"learning_rate": 2.942691603548416e-08,
"logits/chosen": 0.5405811071395874,
"logits/rejected": 1.0355885028839111,
"logps/chosen": -320.9105224609375,
"logps/pi_response": -135.04757690429688,
"logps/ref_response": -127.4443588256836,
"logps/rejected": -553.2946166992188,
"loss": 0.5056,
"rewards/accuracies": 0.7875000238418579,
"rewards/chosen": -0.4617777466773987,
"rewards/margins": 0.8675910234451294,
"rewards/rejected": -1.3293688297271729,
"step": 130
},
{
"epoch": 0.8791208791208791,
"grad_norm": 11.32561508697925,
"learning_rate": 1.2878971655412513e-08,
"logits/chosen": 0.5967448353767395,
"logits/rejected": 0.9229636192321777,
"logps/chosen": -296.9305725097656,
"logps/pi_response": -139.7073974609375,
"logps/ref_response": -129.02000427246094,
"logps/rejected": -566.9810791015625,
"loss": 0.5117,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.4712587893009186,
"rewards/margins": 0.9010453224182129,
"rewards/rejected": -1.3723042011260986,
"step": 140
},
{
"epoch": 0.9419152276295133,
"grad_norm": 11.652819230057418,
"learning_rate": 2.922527618666465e-09,
"logits/chosen": 0.5724425315856934,
"logits/rejected": 0.9267145395278931,
"logps/chosen": -312.5321350097656,
"logps/pi_response": -124.59476470947266,
"logps/ref_response": -114.87628173828125,
"logps/rejected": -503.21697998046875,
"loss": 0.5304,
"rewards/accuracies": 0.7437499761581421,
"rewards/chosen": -0.5373696088790894,
"rewards/margins": 0.6669692993164062,
"rewards/rejected": -1.204338788986206,
"step": 150
},
{
"epoch": 0.9984301412872841,
"step": 159,
"total_flos": 0.0,
"train_loss": 0.557155561147246,
"train_runtime": 4462.021,
"train_samples_per_second": 4.567,
"train_steps_per_second": 0.036
}
],
"logging_steps": 10,
"max_steps": 159,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}
|