| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 73.65079365079364, | |
| "eval_steps": 40, | |
| "global_step": 1160, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 7.928347587585449, | |
| "learning_rate": 7.5e-08, | |
| "logits/chosen": -2.7277705669403076, | |
| "logits/rejected": -2.7679762840270996, | |
| "logps/chosen": -128.4798583984375, | |
| "logps/rejected": -98.36178588867188, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.109375, | |
| "rewards/chosen": -7.17043731128797e-05, | |
| "rewards/margins": 5.325676465872675e-05, | |
| "rewards/rejected": -0.0001249611668754369, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "grad_norm": 9.242063522338867, | |
| "learning_rate": 1.75e-07, | |
| "logits/chosen": -2.6805009841918945, | |
| "logits/rejected": -2.712625026702881, | |
| "logps/chosen": -138.11959838867188, | |
| "logps/rejected": -96.05882263183594, | |
| "loss": 0.693, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.0008271098486147821, | |
| "rewards/margins": 0.00034989125560969114, | |
| "rewards/rejected": 0.00047721865121275187, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "grad_norm": 8.156970977783203, | |
| "learning_rate": 2.75e-07, | |
| "logits/chosen": -2.7695400714874268, | |
| "logits/rejected": -2.7420883178710938, | |
| "logps/chosen": -133.04307556152344, | |
| "logps/rejected": -104.19400024414062, | |
| "loss": 0.6919, | |
| "rewards/accuracies": 0.609375, | |
| "rewards/chosen": 0.0034509659744799137, | |
| "rewards/margins": 0.0025641382671892643, | |
| "rewards/rejected": 0.0008868275908753276, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 1.02, | |
| "grad_norm": 8.723258018493652, | |
| "learning_rate": 3.75e-07, | |
| "logits/chosen": -2.6869003772735596, | |
| "logits/rejected": -2.7081785202026367, | |
| "logps/chosen": -126.2979736328125, | |
| "logps/rejected": -98.4257583618164, | |
| "loss": 0.6889, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": 0.011365305632352829, | |
| "rewards/margins": 0.008443659171462059, | |
| "rewards/rejected": 0.0029216469265520573, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 1.27, | |
| "grad_norm": 10.34566593170166, | |
| "learning_rate": 4.7499999999999995e-07, | |
| "logits/chosen": -2.704402208328247, | |
| "logits/rejected": -2.764737367630005, | |
| "logps/chosen": -125.91941833496094, | |
| "logps/rejected": -99.02344512939453, | |
| "loss": 0.6832, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.02058127522468567, | |
| "rewards/margins": 0.020110297948122025, | |
| "rewards/rejected": 0.00047097797505557537, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 1.52, | |
| "grad_norm": 9.291991233825684, | |
| "learning_rate": 4.999985633126757e-07, | |
| "logits/chosen": -2.716165781021118, | |
| "logits/rejected": -2.6845083236694336, | |
| "logps/chosen": -137.2718505859375, | |
| "logps/rejected": -97.50873565673828, | |
| "loss": 0.6785, | |
| "rewards/accuracies": 0.90625, | |
| "rewards/chosen": 0.03298673778772354, | |
| "rewards/margins": 0.029706846922636032, | |
| "rewards/rejected": 0.0032798887696117163, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 1.78, | |
| "grad_norm": 9.161444664001465, | |
| "learning_rate": 4.999921780689761e-07, | |
| "logits/chosen": -2.735781192779541, | |
| "logits/rejected": -2.7485365867614746, | |
| "logps/chosen": -128.5098876953125, | |
| "logps/rejected": -90.1419906616211, | |
| "loss": 0.6693, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.04758748784661293, | |
| "rewards/margins": 0.04847026616334915, | |
| "rewards/rejected": -0.0008827749406918883, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 2.03, | |
| "grad_norm": 8.495955467224121, | |
| "learning_rate": 4.99980684767309e-07, | |
| "logits/chosen": -2.7206833362579346, | |
| "logits/rejected": -2.750042676925659, | |
| "logps/chosen": -133.54827880859375, | |
| "logps/rejected": -108.23342895507812, | |
| "loss": 0.6669, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.0631466805934906, | |
| "rewards/margins": 0.05378420650959015, | |
| "rewards/rejected": 0.009362474083900452, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 2.29, | |
| "grad_norm": 8.951277732849121, | |
| "learning_rate": 4.999640836425158e-07, | |
| "logits/chosen": -2.6706807613372803, | |
| "logits/rejected": -2.6151680946350098, | |
| "logps/chosen": -136.55226135253906, | |
| "logps/rejected": -97.76315307617188, | |
| "loss": 0.6578, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.08149851858615875, | |
| "rewards/margins": 0.07302143424749374, | |
| "rewards/rejected": 0.008477086201310158, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 2.54, | |
| "grad_norm": 8.608317375183105, | |
| "learning_rate": 4.99942375033805e-07, | |
| "logits/chosen": -2.7676639556884766, | |
| "logits/rejected": -2.833590030670166, | |
| "logps/chosen": -133.7703857421875, | |
| "logps/rejected": -106.12245178222656, | |
| "loss": 0.6488, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.10222765803337097, | |
| "rewards/margins": 0.09170522540807724, | |
| "rewards/rejected": 0.010522443801164627, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 2.54, | |
| "eval_logits/chosen": -2.806352138519287, | |
| "eval_logits/rejected": -2.8898861408233643, | |
| "eval_logps/chosen": -133.14093017578125, | |
| "eval_logps/rejected": -97.8843002319336, | |
| "eval_loss": 0.6592782139778137, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 0.09283973276615143, | |
| "eval_rewards/margins": 0.07038124650716782, | |
| "eval_rewards/rejected": 0.022458484396338463, | |
| "eval_runtime": 19.3683, | |
| "eval_samples_per_second": 1.446, | |
| "eval_steps_per_second": 1.446, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 2.79, | |
| "grad_norm": 9.09485912322998, | |
| "learning_rate": 4.999155593847457e-07, | |
| "logits/chosen": -2.7024385929107666, | |
| "logits/rejected": -2.7564737796783447, | |
| "logps/chosen": -123.412353515625, | |
| "logps/rejected": -91.91966247558594, | |
| "loss": 0.6347, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.114817313849926, | |
| "rewards/margins": 0.12200291454792023, | |
| "rewards/rejected": -0.007185595575720072, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 3.05, | |
| "grad_norm": 9.453149795532227, | |
| "learning_rate": 4.998836372432589e-07, | |
| "logits/chosen": -2.736665725708008, | |
| "logits/rejected": -2.757841110229492, | |
| "logps/chosen": -127.0240478515625, | |
| "logps/rejected": -103.66780853271484, | |
| "loss": 0.636, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.13144327700138092, | |
| "rewards/margins": 0.11972987651824951, | |
| "rewards/rejected": 0.011713397689163685, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 3.3, | |
| "grad_norm": 8.674492835998535, | |
| "learning_rate": 4.998466092616052e-07, | |
| "logits/chosen": -2.710110664367676, | |
| "logits/rejected": -2.7205581665039062, | |
| "logps/chosen": -131.0464324951172, | |
| "logps/rejected": -99.71279907226562, | |
| "loss": 0.6166, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.15897569060325623, | |
| "rewards/margins": 0.16229701042175293, | |
| "rewards/rejected": -0.003321333322674036, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 3.56, | |
| "grad_norm": 9.434328079223633, | |
| "learning_rate": 4.99804476196373e-07, | |
| "logits/chosen": -2.685814380645752, | |
| "logits/rejected": -2.736269474029541, | |
| "logps/chosen": -125.750732421875, | |
| "logps/rejected": -93.69428253173828, | |
| "loss": 0.6131, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.16781488060951233, | |
| "rewards/margins": 0.16998659074306488, | |
| "rewards/rejected": -0.002171688713133335, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 3.81, | |
| "grad_norm": 8.643455505371094, | |
| "learning_rate": 4.997572389084615e-07, | |
| "logits/chosen": -2.7429587841033936, | |
| "logits/rejected": -2.75490140914917, | |
| "logps/chosen": -131.4405517578125, | |
| "logps/rejected": -99.93754577636719, | |
| "loss": 0.614, | |
| "rewards/accuracies": 0.921875, | |
| "rewards/chosen": 0.18873561918735504, | |
| "rewards/margins": 0.17132429778575897, | |
| "rewards/rejected": 0.01741132140159607, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 4.06, | |
| "grad_norm": 8.412778854370117, | |
| "learning_rate": 4.997048983630643e-07, | |
| "logits/chosen": -2.6886143684387207, | |
| "logits/rejected": -2.710732936859131, | |
| "logps/chosen": -123.77167510986328, | |
| "logps/rejected": -97.30846405029297, | |
| "loss": 0.6099, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.19313779473304749, | |
| "rewards/margins": 0.1793561577796936, | |
| "rewards/rejected": 0.013781649991869926, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 4.32, | |
| "grad_norm": 7.851288795471191, | |
| "learning_rate": 4.99647455629649e-07, | |
| "logits/chosen": -2.744528293609619, | |
| "logits/rejected": -2.7117881774902344, | |
| "logps/chosen": -136.25863647460938, | |
| "logps/rejected": -95.11305236816406, | |
| "loss": 0.5968, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.21463464200496674, | |
| "rewards/margins": 0.20793242752552032, | |
| "rewards/rejected": 0.0067022331058979034, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 4.57, | |
| "grad_norm": 9.278959274291992, | |
| "learning_rate": 4.995849118819353e-07, | |
| "logits/chosen": -2.7611894607543945, | |
| "logits/rejected": -2.7447633743286133, | |
| "logps/chosen": -134.3462677001953, | |
| "logps/rejected": -103.75216674804688, | |
| "loss": 0.5828, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 0.25433310866355896, | |
| "rewards/margins": 0.24298511445522308, | |
| "rewards/rejected": 0.011347985826432705, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 4.83, | |
| "grad_norm": 7.795536994934082, | |
| "learning_rate": 4.995172683978719e-07, | |
| "logits/chosen": -2.698429584503174, | |
| "logits/rejected": -2.7458348274230957, | |
| "logps/chosen": -127.65770721435547, | |
| "logps/rejected": -104.23495483398438, | |
| "loss": 0.5796, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.25434932112693787, | |
| "rewards/margins": 0.25114190578460693, | |
| "rewards/rejected": 0.0032073920592665672, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 5.08, | |
| "grad_norm": 7.788575649261475, | |
| "learning_rate": 4.994445265596091e-07, | |
| "logits/chosen": -2.6734955310821533, | |
| "logits/rejected": -2.7292165756225586, | |
| "logps/chosen": -123.53501892089844, | |
| "logps/rejected": -95.23420715332031, | |
| "loss": 0.5676, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.2950035631656647, | |
| "rewards/margins": 0.2805778682231903, | |
| "rewards/rejected": 0.014425676316022873, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 5.08, | |
| "eval_logits/chosen": -2.8074097633361816, | |
| "eval_logits/rejected": -2.890533685684204, | |
| "eval_logps/chosen": -131.55104064941406, | |
| "eval_logps/rejected": -97.56663513183594, | |
| "eval_loss": 0.6045485734939575, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 0.2518289387226105, | |
| "eval_rewards/margins": 0.1976027488708496, | |
| "eval_rewards/rejected": 0.05422618240118027, | |
| "eval_runtime": 19.3586, | |
| "eval_samples_per_second": 1.446, | |
| "eval_steps_per_second": 1.446, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 5.33, | |
| "grad_norm": 6.709670543670654, | |
| "learning_rate": 4.993666878534718e-07, | |
| "logits/chosen": -2.736870527267456, | |
| "logits/rejected": -2.778140068054199, | |
| "logps/chosen": -128.86972045898438, | |
| "logps/rejected": -104.70977020263672, | |
| "loss": 0.5637, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.31255263090133667, | |
| "rewards/margins": 0.2908375561237335, | |
| "rewards/rejected": 0.021715058013796806, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 5.59, | |
| "grad_norm": 8.178019523620605, | |
| "learning_rate": 4.99283753869928e-07, | |
| "logits/chosen": -2.686039447784424, | |
| "logits/rejected": -2.681716203689575, | |
| "logps/chosen": -129.52798461914062, | |
| "logps/rejected": -97.80433654785156, | |
| "loss": 0.5475, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.32084640860557556, | |
| "rewards/margins": 0.33009591698646545, | |
| "rewards/rejected": -0.00924946740269661, | |
| "step": 88 | |
| }, | |
| { | |
| "epoch": 5.84, | |
| "grad_norm": 7.515178680419922, | |
| "learning_rate": 4.991957263035573e-07, | |
| "logits/chosen": -2.763765811920166, | |
| "logits/rejected": -2.7660324573516846, | |
| "logps/chosen": -128.16851806640625, | |
| "logps/rejected": -95.72579956054688, | |
| "loss": 0.5493, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.3080848455429077, | |
| "rewards/margins": 0.32350555062294006, | |
| "rewards/rejected": -0.015420722775161266, | |
| "step": 92 | |
| }, | |
| { | |
| "epoch": 6.1, | |
| "grad_norm": 8.599517822265625, | |
| "learning_rate": 4.991026069530156e-07, | |
| "logits/chosen": -2.7107338905334473, | |
| "logits/rejected": -2.745981216430664, | |
| "logps/chosen": -131.5907745361328, | |
| "logps/rejected": -102.65729522705078, | |
| "loss": 0.5568, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.36698901653289795, | |
| "rewards/margins": 0.3141450881958008, | |
| "rewards/rejected": 0.05284389108419418, | |
| "step": 96 | |
| }, | |
| { | |
| "epoch": 6.35, | |
| "grad_norm": 7.087887287139893, | |
| "learning_rate": 4.990043977209984e-07, | |
| "logits/chosen": -2.75876522064209, | |
| "logits/rejected": -2.800281047821045, | |
| "logps/chosen": -124.87574005126953, | |
| "logps/rejected": -99.60267639160156, | |
| "loss": 0.5332, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.3760470151901245, | |
| "rewards/margins": 0.37043848633766174, | |
| "rewards/rejected": 0.00560858054086566, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 6.6, | |
| "grad_norm": 6.458583831787109, | |
| "learning_rate": 4.989011006142024e-07, | |
| "logits/chosen": -2.70744252204895, | |
| "logits/rejected": -2.7285239696502686, | |
| "logps/chosen": -129.41262817382812, | |
| "logps/rejected": -98.60986328125, | |
| "loss": 0.5286, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 0.3909401297569275, | |
| "rewards/margins": 0.3845981955528259, | |
| "rewards/rejected": 0.006341893225908279, | |
| "step": 104 | |
| }, | |
| { | |
| "epoch": 6.86, | |
| "grad_norm": 8.084393501281738, | |
| "learning_rate": 4.987927177432842e-07, | |
| "logits/chosen": -2.731127977371216, | |
| "logits/rejected": -2.7352988719940186, | |
| "logps/chosen": -123.85149383544922, | |
| "logps/rejected": -97.99423217773438, | |
| "loss": 0.544, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": 0.38825392723083496, | |
| "rewards/margins": 0.34452566504478455, | |
| "rewards/rejected": 0.043728262186050415, | |
| "step": 108 | |
| }, | |
| { | |
| "epoch": 7.11, | |
| "grad_norm": 7.936505317687988, | |
| "learning_rate": 4.987080943856886e-07, | |
| "logits/chosen": -2.6931982040405273, | |
| "logits/rejected": -2.683288812637329, | |
| "logps/chosen": -130.97227478027344, | |
| "logps/rejected": -93.71176147460938, | |
| "loss": 0.4903, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.45238468050956726, | |
| "rewards/margins": 0.48204928636550903, | |
| "rewards/rejected": -0.029664600268006325, | |
| "step": 112 | |
| }, | |
| { | |
| "epoch": 7.37, | |
| "grad_norm": 6.827272415161133, | |
| "learning_rate": 4.985908168188602e-07, | |
| "logits/chosen": -2.697653293609619, | |
| "logits/rejected": -2.7467117309570312, | |
| "logps/chosen": -119.16637420654297, | |
| "logps/rejected": -95.0647201538086, | |
| "loss": 0.5088, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.4183903634548187, | |
| "rewards/margins": 0.43436184525489807, | |
| "rewards/rejected": -0.015971507877111435, | |
| "step": 116 | |
| }, | |
| { | |
| "epoch": 7.62, | |
| "grad_norm": 6.599144458770752, | |
| "learning_rate": 4.984684598278982e-07, | |
| "logits/chosen": -2.7136449813842773, | |
| "logits/rejected": -2.7430026531219482, | |
| "logps/chosen": -126.032958984375, | |
| "logps/rejected": -102.69947814941406, | |
| "loss": 0.5231, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.4575308859348297, | |
| "rewards/margins": 0.40544256567955017, | |
| "rewards/rejected": 0.052088312804698944, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 7.62, | |
| "eval_logits/chosen": -2.807668447494507, | |
| "eval_logits/rejected": -2.8901402950286865, | |
| "eval_logps/chosen": -129.97122192382812, | |
| "eval_logps/rejected": -97.28166961669922, | |
| "eval_loss": 0.5570675134658813, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 0.4098115563392639, | |
| "eval_rewards/margins": 0.3270883560180664, | |
| "eval_rewards/rejected": 0.0827232226729393, | |
| "eval_runtime": 19.4265, | |
| "eval_samples_per_second": 1.441, | |
| "eval_steps_per_second": 1.441, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 7.87, | |
| "grad_norm": 7.769857883453369, | |
| "learning_rate": 4.983410259129073e-07, | |
| "logits/chosen": -2.7467093467712402, | |
| "logits/rejected": -2.7454986572265625, | |
| "logps/chosen": -134.60739135742188, | |
| "logps/rejected": -103.20536804199219, | |
| "loss": 0.4995, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 0.4985056221485138, | |
| "rewards/margins": 0.4699307978153229, | |
| "rewards/rejected": 0.028574790805578232, | |
| "step": 124 | |
| }, | |
| { | |
| "epoch": 8.13, | |
| "grad_norm": 7.224113941192627, | |
| "learning_rate": 4.982085176777285e-07, | |
| "logits/chosen": -2.725543975830078, | |
| "logits/rejected": -2.7536404132843018, | |
| "logps/chosen": -121.35929870605469, | |
| "logps/rejected": -97.49042510986328, | |
| "loss": 0.4988, | |
| "rewards/accuracies": 0.921875, | |
| "rewards/chosen": 0.4724448025226593, | |
| "rewards/margins": 0.4719679057598114, | |
| "rewards/rejected": 0.0004768962971866131, | |
| "step": 128 | |
| }, | |
| { | |
| "epoch": 8.38, | |
| "grad_norm": 5.916596412658691, | |
| "learning_rate": 4.980709378298851e-07, | |
| "logits/chosen": -2.7414817810058594, | |
| "logits/rejected": -2.7378463745117188, | |
| "logps/chosen": -119.76469421386719, | |
| "logps/rejected": -92.40216827392578, | |
| "loss": 0.4946, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 0.48038381338119507, | |
| "rewards/margins": 0.4809207320213318, | |
| "rewards/rejected": -0.0005369335412979126, | |
| "step": 132 | |
| }, | |
| { | |
| "epoch": 8.63, | |
| "grad_norm": 6.414724826812744, | |
| "learning_rate": 4.979282891805286e-07, | |
| "logits/chosen": -2.7029452323913574, | |
| "logits/rejected": -2.737753391265869, | |
| "logps/chosen": -130.59829711914062, | |
| "logps/rejected": -99.5873794555664, | |
| "loss": 0.4564, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.5609064698219299, | |
| "rewards/margins": 0.5927521586418152, | |
| "rewards/rejected": -0.03184571862220764, | |
| "step": 136 | |
| }, | |
| { | |
| "epoch": 8.89, | |
| "grad_norm": 7.006284236907959, | |
| "learning_rate": 4.977805746443806e-07, | |
| "logits/chosen": -2.719618320465088, | |
| "logits/rejected": -2.7250468730926514, | |
| "logps/chosen": -128.04678344726562, | |
| "logps/rejected": -101.81006622314453, | |
| "loss": 0.4761, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.5477940440177917, | |
| "rewards/margins": 0.5303957462310791, | |
| "rewards/rejected": 0.017398254945874214, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 9.14, | |
| "grad_norm": 7.040762901306152, | |
| "learning_rate": 4.97627797239673e-07, | |
| "logits/chosen": -2.6895217895507812, | |
| "logits/rejected": -2.7162435054779053, | |
| "logps/chosen": -133.3381805419922, | |
| "logps/rejected": -102.25975799560547, | |
| "loss": 0.4403, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.6185128688812256, | |
| "rewards/margins": 0.6498669981956482, | |
| "rewards/rejected": -0.03135409206151962, | |
| "step": 144 | |
| }, | |
| { | |
| "epoch": 9.4, | |
| "grad_norm": 6.93245267868042, | |
| "learning_rate": 4.974699600880869e-07, | |
| "logits/chosen": -2.6911263465881348, | |
| "logits/rejected": -2.7075552940368652, | |
| "logps/chosen": -126.95785522460938, | |
| "logps/rejected": -102.03314971923828, | |
| "loss": 0.4587, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.573100745677948, | |
| "rewards/margins": 0.5804403424263, | |
| "rewards/rejected": -0.0073395660147070885, | |
| "step": 148 | |
| }, | |
| { | |
| "epoch": 9.65, | |
| "grad_norm": 7.084275245666504, | |
| "learning_rate": 4.973070664146885e-07, | |
| "logits/chosen": -2.7053210735321045, | |
| "logits/rejected": -2.7612156867980957, | |
| "logps/chosen": -119.92796325683594, | |
| "logps/rejected": -97.49836730957031, | |
| "loss": 0.483, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.5761281251907349, | |
| "rewards/margins": 0.5190825462341309, | |
| "rewards/rejected": 0.057045597583055496, | |
| "step": 152 | |
| }, | |
| { | |
| "epoch": 9.9, | |
| "grad_norm": 5.3888258934021, | |
| "learning_rate": 4.971391195478631e-07, | |
| "logits/chosen": -2.750107765197754, | |
| "logits/rejected": -2.7442452907562256, | |
| "logps/chosen": -128.85630798339844, | |
| "logps/rejected": -96.89845275878906, | |
| "loss": 0.4517, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.6420415043830872, | |
| "rewards/margins": 0.6197397708892822, | |
| "rewards/rejected": 0.022301698103547096, | |
| "step": 156 | |
| }, | |
| { | |
| "epoch": 10.16, | |
| "grad_norm": 6.74313497543335, | |
| "learning_rate": 4.969661229192477e-07, | |
| "logits/chosen": -2.7252540588378906, | |
| "logits/rejected": -2.709249258041382, | |
| "logps/chosen": -126.21170043945312, | |
| "logps/rejected": -105.25879669189453, | |
| "loss": 0.4607, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 0.6133719682693481, | |
| "rewards/margins": 0.6076129674911499, | |
| "rewards/rejected": 0.005758981220424175, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 10.16, | |
| "eval_logits/chosen": -2.807128429412842, | |
| "eval_logits/rejected": -2.8889076709747314, | |
| "eval_logps/chosen": -128.48606872558594, | |
| "eval_logps/rejected": -97.06517028808594, | |
| "eval_loss": 0.5175719857215881, | |
| "eval_rewards/accuracies": 0.8214285969734192, | |
| "eval_rewards/chosen": 0.5583271384239197, | |
| "eval_rewards/margins": 0.4539553225040436, | |
| "eval_rewards/rejected": 0.10437185317277908, | |
| "eval_runtime": 19.3848, | |
| "eval_samples_per_second": 1.444, | |
| "eval_steps_per_second": 1.444, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 10.41, | |
| "grad_norm": 5.443469524383545, | |
| "learning_rate": 4.967880800636598e-07, | |
| "logits/chosen": -2.7496747970581055, | |
| "logits/rejected": -2.768813133239746, | |
| "logps/chosen": -122.11544799804688, | |
| "logps/rejected": -94.54540252685547, | |
| "loss": 0.436, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.6616504788398743, | |
| "rewards/margins": 0.67127525806427, | |
| "rewards/rejected": -0.009624744765460491, | |
| "step": 164 | |
| }, | |
| { | |
| "epoch": 10.67, | |
| "grad_norm": 5.551913738250732, | |
| "learning_rate": 4.966049946190263e-07, | |
| "logits/chosen": -2.7052814960479736, | |
| "logits/rejected": -2.7216036319732666, | |
| "logps/chosen": -131.8173065185547, | |
| "logps/rejected": -102.79641723632812, | |
| "loss": 0.4326, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.7013098001480103, | |
| "rewards/margins": 0.6718720197677612, | |
| "rewards/rejected": 0.02943783439695835, | |
| "step": 168 | |
| }, | |
| { | |
| "epoch": 10.92, | |
| "grad_norm": 5.826549530029297, | |
| "learning_rate": 4.964168703263086e-07, | |
| "logits/chosen": -2.701007604598999, | |
| "logits/rejected": -2.741464614868164, | |
| "logps/chosen": -116.08711242675781, | |
| "logps/rejected": -94.73246765136719, | |
| "loss": 0.446, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.6390998363494873, | |
| "rewards/margins": 0.6307864785194397, | |
| "rewards/rejected": 0.008313396014273167, | |
| "step": 172 | |
| }, | |
| { | |
| "epoch": 11.17, | |
| "grad_norm": 6.066051483154297, | |
| "learning_rate": 4.962237110294259e-07, | |
| "logits/chosen": -2.7041149139404297, | |
| "logits/rejected": -2.731684684753418, | |
| "logps/chosen": -124.84115600585938, | |
| "logps/rejected": -103.8166732788086, | |
| "loss": 0.4294, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.6835889220237732, | |
| "rewards/margins": 0.6921644806861877, | |
| "rewards/rejected": -0.00857558287680149, | |
| "step": 176 | |
| }, | |
| { | |
| "epoch": 11.43, | |
| "grad_norm": 5.8145012855529785, | |
| "learning_rate": 4.960255206751773e-07, | |
| "logits/chosen": -2.7352046966552734, | |
| "logits/rejected": -2.7856600284576416, | |
| "logps/chosen": -123.47644805908203, | |
| "logps/rejected": -93.96688079833984, | |
| "loss": 0.4056, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.7594637274742126, | |
| "rewards/margins": 0.7790957093238831, | |
| "rewards/rejected": -0.019631966948509216, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 11.68, | |
| "grad_norm": 5.962335109710693, | |
| "learning_rate": 4.958223033131609e-07, | |
| "logits/chosen": -2.740914821624756, | |
| "logits/rejected": -2.7585391998291016, | |
| "logps/chosen": -124.35005950927734, | |
| "logps/rejected": -105.05398559570312, | |
| "loss": 0.4293, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 0.6812430620193481, | |
| "rewards/margins": 0.6962765455245972, | |
| "rewards/rejected": -0.015033497475087643, | |
| "step": 184 | |
| }, | |
| { | |
| "epoch": 11.94, | |
| "grad_norm": 5.529379367828369, | |
| "learning_rate": 4.956140630956907e-07, | |
| "logits/chosen": -2.7024734020233154, | |
| "logits/rejected": -2.6746203899383545, | |
| "logps/chosen": -125.29593658447266, | |
| "logps/rejected": -94.42231750488281, | |
| "loss": 0.3963, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 0.7892162799835205, | |
| "rewards/margins": 0.8050926327705383, | |
| "rewards/rejected": -0.01587628945708275, | |
| "step": 188 | |
| }, | |
| { | |
| "epoch": 12.19, | |
| "grad_norm": 5.774294376373291, | |
| "learning_rate": 4.954008042777125e-07, | |
| "logits/chosen": -2.7662813663482666, | |
| "logits/rejected": -2.8167452812194824, | |
| "logps/chosen": -124.10614776611328, | |
| "logps/rejected": -94.9889907836914, | |
| "loss": 0.3728, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.8182777166366577, | |
| "rewards/margins": 0.8937400579452515, | |
| "rewards/rejected": -0.07546230405569077, | |
| "step": 192 | |
| }, | |
| { | |
| "epoch": 12.44, | |
| "grad_norm": 4.990906715393066, | |
| "learning_rate": 4.951825312167159e-07, | |
| "logits/chosen": -2.7352771759033203, | |
| "logits/rejected": -2.7206149101257324, | |
| "logps/chosen": -116.98539733886719, | |
| "logps/rejected": -96.68843078613281, | |
| "loss": 0.4435, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.7208267450332642, | |
| "rewards/margins": 0.6662319302558899, | |
| "rewards/rejected": 0.05459484085440636, | |
| "step": 196 | |
| }, | |
| { | |
| "epoch": 12.7, | |
| "grad_norm": 7.139024257659912, | |
| "learning_rate": 4.949592483726464e-07, | |
| "logits/chosen": -2.728516101837158, | |
| "logits/rejected": -2.732177257537842, | |
| "logps/chosen": -125.51838684082031, | |
| "logps/rejected": -103.7790298461914, | |
| "loss": 0.391, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 0.7955451011657715, | |
| "rewards/margins": 0.8273122310638428, | |
| "rewards/rejected": -0.03176717832684517, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 12.7, | |
| "eval_logits/chosen": -2.8053383827209473, | |
| "eval_logits/rejected": -2.8861937522888184, | |
| "eval_logps/chosen": -127.08057403564453, | |
| "eval_logps/rejected": -96.8802261352539, | |
| "eval_loss": 0.4858916103839874, | |
| "eval_rewards/accuracies": 0.8214285969734192, | |
| "eval_rewards/chosen": 0.6988765597343445, | |
| "eval_rewards/margins": 0.5760089755058289, | |
| "eval_rewards/rejected": 0.12286762148141861, | |
| "eval_runtime": 19.4173, | |
| "eval_samples_per_second": 1.442, | |
| "eval_steps_per_second": 1.442, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 12.95, | |
| "grad_norm": 5.450782299041748, | |
| "learning_rate": 4.947309603078138e-07, | |
| "logits/chosen": -2.6719353199005127, | |
| "logits/rejected": -2.7134647369384766, | |
| "logps/chosen": -124.22660827636719, | |
| "logps/rejected": -99.73461151123047, | |
| "loss": 0.4062, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.7935754060745239, | |
| "rewards/margins": 0.783437192440033, | |
| "rewards/rejected": 0.010138224810361862, | |
| "step": 204 | |
| }, | |
| { | |
| "epoch": 13.21, | |
| "grad_norm": 5.752992153167725, | |
| "learning_rate": 4.944976716867984e-07, | |
| "logits/chosen": -2.7026755809783936, | |
| "logits/rejected": -2.698920726776123, | |
| "logps/chosen": -123.63475036621094, | |
| "logps/rejected": -107.06332397460938, | |
| "loss": 0.3952, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.8116875886917114, | |
| "rewards/margins": 0.8335397839546204, | |
| "rewards/rejected": -0.02185220457613468, | |
| "step": 208 | |
| }, | |
| { | |
| "epoch": 13.46, | |
| "grad_norm": 5.456747531890869, | |
| "learning_rate": 4.942593872763566e-07, | |
| "logits/chosen": -2.701968193054199, | |
| "logits/rejected": -2.786785840988159, | |
| "logps/chosen": -124.91243743896484, | |
| "logps/rejected": -101.95451354980469, | |
| "loss": 0.3673, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.8831655979156494, | |
| "rewards/margins": 0.9259040951728821, | |
| "rewards/rejected": -0.04273851588368416, | |
| "step": 212 | |
| }, | |
| { | |
| "epoch": 13.71, | |
| "grad_norm": 4.753533363342285, | |
| "learning_rate": 4.940161119453231e-07, | |
| "logits/chosen": -2.7109193801879883, | |
| "logits/rejected": -2.6961448192596436, | |
| "logps/chosen": -123.21333312988281, | |
| "logps/rejected": -92.74653625488281, | |
| "loss": 0.3832, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 0.8407853245735168, | |
| "rewards/margins": 0.8698095083236694, | |
| "rewards/rejected": -0.02902432158589363, | |
| "step": 216 | |
| }, | |
| { | |
| "epoch": 13.97, | |
| "grad_norm": 5.374657154083252, | |
| "learning_rate": 4.937678506645115e-07, | |
| "logits/chosen": -2.7288832664489746, | |
| "logits/rejected": -2.7349352836608887, | |
| "logps/chosen": -123.1498031616211, | |
| "logps/rejected": -95.77909851074219, | |
| "loss": 0.3644, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 0.8925550580024719, | |
| "rewards/margins": 0.9183064699172974, | |
| "rewards/rejected": -0.025751333683729172, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 14.22, | |
| "grad_norm": 5.842701435089111, | |
| "learning_rate": 4.935146085066125e-07, | |
| "logits/chosen": -2.7030577659606934, | |
| "logits/rejected": -2.7200875282287598, | |
| "logps/chosen": -120.81951904296875, | |
| "logps/rejected": -100.53494262695312, | |
| "loss": 0.3581, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.9156714677810669, | |
| "rewards/margins": 0.9704564809799194, | |
| "rewards/rejected": -0.05478499457240105, | |
| "step": 224 | |
| }, | |
| { | |
| "epoch": 14.48, | |
| "grad_norm": 5.020945072174072, | |
| "learning_rate": 4.932563906460904e-07, | |
| "logits/chosen": -2.722069501876831, | |
| "logits/rejected": -2.731358051300049, | |
| "logps/chosen": -123.65387725830078, | |
| "logps/rejected": -103.83390808105469, | |
| "loss": 0.391, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.8787897229194641, | |
| "rewards/margins": 0.8743190765380859, | |
| "rewards/rejected": 0.004470685496926308, | |
| "step": 228 | |
| }, | |
| { | |
| "epoch": 14.73, | |
| "grad_norm": 5.041106700897217, | |
| "learning_rate": 4.929932023590776e-07, | |
| "logits/chosen": -2.735006093978882, | |
| "logits/rejected": -2.7546191215515137, | |
| "logps/chosen": -124.65998840332031, | |
| "logps/rejected": -101.49640655517578, | |
| "loss": 0.3586, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.9206160306930542, | |
| "rewards/margins": 0.9452594518661499, | |
| "rewards/rejected": -0.024643439799547195, | |
| "step": 232 | |
| }, | |
| { | |
| "epoch": 14.98, | |
| "grad_norm": 5.318466663360596, | |
| "learning_rate": 4.927250490232664e-07, | |
| "logits/chosen": -2.6883015632629395, | |
| "logits/rejected": -2.718212842941284, | |
| "logps/chosen": -120.88870239257812, | |
| "logps/rejected": -94.79769134521484, | |
| "loss": 0.3564, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.9339860677719116, | |
| "rewards/margins": 0.9901734590530396, | |
| "rewards/rejected": -0.05618742108345032, | |
| "step": 236 | |
| }, | |
| { | |
| "epoch": 15.24, | |
| "grad_norm": 5.438621997833252, | |
| "learning_rate": 4.924519361177992e-07, | |
| "logits/chosen": -2.7316794395446777, | |
| "logits/rejected": -2.812800407409668, | |
| "logps/chosen": -120.06542205810547, | |
| "logps/rejected": -97.61270141601562, | |
| "loss": 0.3471, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.9851706624031067, | |
| "rewards/margins": 1.039989948272705, | |
| "rewards/rejected": -0.05481935292482376, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 15.24, | |
| "eval_logits/chosen": -2.8022568225860596, | |
| "eval_logits/rejected": -2.882326602935791, | |
| "eval_logps/chosen": -125.87542724609375, | |
| "eval_logps/rejected": -96.83832550048828, | |
| "eval_loss": 0.46029001474380493, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 0.8193913102149963, | |
| "eval_rewards/margins": 0.6923348307609558, | |
| "eval_rewards/rejected": 0.12705650925636292, | |
| "eval_runtime": 19.4065, | |
| "eval_samples_per_second": 1.443, | |
| "eval_steps_per_second": 1.443, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 15.49, | |
| "grad_norm": 5.301855087280273, | |
| "learning_rate": 4.921738692231572e-07, | |
| "logits/chosen": -2.661013126373291, | |
| "logits/rejected": -2.666978359222412, | |
| "logps/chosen": -121.69336700439453, | |
| "logps/rejected": -101.86541748046875, | |
| "loss": 0.3574, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 0.9682250022888184, | |
| "rewards/margins": 0.9734476208686829, | |
| "rewards/rejected": -0.0052225226536393166, | |
| "step": 244 | |
| }, | |
| { | |
| "epoch": 15.75, | |
| "grad_norm": 4.478713512420654, | |
| "learning_rate": 4.918908540210451e-07, | |
| "logits/chosen": -2.702775239944458, | |
| "logits/rejected": -2.7098357677459717, | |
| "logps/chosen": -119.93941497802734, | |
| "logps/rejected": -94.76625061035156, | |
| "loss": 0.3517, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.8942417502403259, | |
| "rewards/margins": 0.9958410263061523, | |
| "rewards/rejected": -0.10159924626350403, | |
| "step": 248 | |
| }, | |
| { | |
| "epoch": 16.0, | |
| "grad_norm": 4.398434162139893, | |
| "learning_rate": 4.916028962942762e-07, | |
| "logits/chosen": -2.7399990558624268, | |
| "logits/rejected": -2.7476425170898438, | |
| "logps/chosen": -124.2032470703125, | |
| "logps/rejected": -101.86943054199219, | |
| "loss": 0.3372, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.0369476079940796, | |
| "rewards/margins": 1.0664609670639038, | |
| "rewards/rejected": -0.029513416811823845, | |
| "step": 252 | |
| }, | |
| { | |
| "epoch": 16.25, | |
| "grad_norm": 4.430797576904297, | |
| "learning_rate": 4.913100019266536e-07, | |
| "logits/chosen": -2.6950950622558594, | |
| "logits/rejected": -2.7181689739227295, | |
| "logps/chosen": -123.97085571289062, | |
| "logps/rejected": -97.85747528076172, | |
| "loss": 0.3226, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.0282835960388184, | |
| "rewards/margins": 1.1182292699813843, | |
| "rewards/rejected": -0.08994561433792114, | |
| "step": 256 | |
| }, | |
| { | |
| "epoch": 16.51, | |
| "grad_norm": 6.383476734161377, | |
| "learning_rate": 4.910121769028503e-07, | |
| "logits/chosen": -2.7224864959716797, | |
| "logits/rejected": -2.727613925933838, | |
| "logps/chosen": -117.91517639160156, | |
| "logps/rejected": -99.52539825439453, | |
| "loss": 0.3658, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.9342272877693176, | |
| "rewards/margins": 0.9777376651763916, | |
| "rewards/rejected": -0.04351034015417099, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 16.76, | |
| "grad_norm": 4.453210353851318, | |
| "learning_rate": 4.907094273082864e-07, | |
| "logits/chosen": -2.7149100303649902, | |
| "logits/rejected": -2.710146188735962, | |
| "logps/chosen": -126.07495880126953, | |
| "logps/rejected": -100.1856689453125, | |
| "loss": 0.3162, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 1.1231064796447754, | |
| "rewards/margins": 1.175417423248291, | |
| "rewards/rejected": -0.05231098458170891, | |
| "step": 264 | |
| }, | |
| { | |
| "epoch": 17.02, | |
| "grad_norm": 4.718712329864502, | |
| "learning_rate": 4.904017593290055e-07, | |
| "logits/chosen": -2.710063934326172, | |
| "logits/rejected": -2.7535319328308105, | |
| "logps/chosen": -117.01451873779297, | |
| "logps/rejected": -99.95758819580078, | |
| "loss": 0.322, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.033459186553955, | |
| "rewards/margins": 1.1137871742248535, | |
| "rewards/rejected": -0.08032786101102829, | |
| "step": 268 | |
| }, | |
| { | |
| "epoch": 17.27, | |
| "grad_norm": 5.164140701293945, | |
| "learning_rate": 4.900891792515478e-07, | |
| "logits/chosen": -2.7342586517333984, | |
| "logits/rejected": -2.7561776638031006, | |
| "logps/chosen": -118.80335998535156, | |
| "logps/rejected": -104.33666229248047, | |
| "loss": 0.3314, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.078546404838562, | |
| "rewards/margins": 1.0902469158172607, | |
| "rewards/rejected": -0.011700518429279327, | |
| "step": 272 | |
| }, | |
| { | |
| "epoch": 17.52, | |
| "grad_norm": 4.392764568328857, | |
| "learning_rate": 4.897716934628217e-07, | |
| "logits/chosen": -2.6796340942382812, | |
| "logits/rejected": -2.6666793823242188, | |
| "logps/chosen": -122.01671600341797, | |
| "logps/rejected": -97.74385070800781, | |
| "loss": 0.3047, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.0594760179519653, | |
| "rewards/margins": 1.1904582977294922, | |
| "rewards/rejected": -0.1309822052717209, | |
| "step": 276 | |
| }, | |
| { | |
| "epoch": 17.78, | |
| "grad_norm": 4.647542953491211, | |
| "learning_rate": 4.894493084499736e-07, | |
| "logits/chosen": -2.7203075885772705, | |
| "logits/rejected": -2.7299890518188477, | |
| "logps/chosen": -120.08210754394531, | |
| "logps/rejected": -98.8072738647461, | |
| "loss": 0.3496, | |
| "rewards/accuracies": 0.921875, | |
| "rewards/chosen": 1.0658477544784546, | |
| "rewards/margins": 1.065575122833252, | |
| "rewards/rejected": 0.0002726661041378975, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 17.78, | |
| "eval_logits/chosen": -2.798058271408081, | |
| "eval_logits/rejected": -2.8771989345550537, | |
| "eval_logps/chosen": -124.81001281738281, | |
| "eval_logps/rejected": -96.83968353271484, | |
| "eval_loss": 0.44096532464027405, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 0.9259332418441772, | |
| "eval_rewards/margins": 0.799013078212738, | |
| "eval_rewards/rejected": 0.12692023813724518, | |
| "eval_runtime": 19.3925, | |
| "eval_samples_per_second": 1.444, | |
| "eval_steps_per_second": 1.444, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 18.03, | |
| "grad_norm": 4.158952236175537, | |
| "learning_rate": 4.891220308002547e-07, | |
| "logits/chosen": -2.6795365810394287, | |
| "logits/rejected": -2.7270519733428955, | |
| "logps/chosen": -120.39765167236328, | |
| "logps/rejected": -100.02141571044922, | |
| "loss": 0.319, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 1.0584629774093628, | |
| "rewards/margins": 1.19419264793396, | |
| "rewards/rejected": -0.13572952151298523, | |
| "step": 284 | |
| }, | |
| { | |
| "epoch": 18.29, | |
| "grad_norm": 4.605132579803467, | |
| "learning_rate": 4.887898672008871e-07, | |
| "logits/chosen": -2.696953296661377, | |
| "logits/rejected": -2.718316078186035, | |
| "logps/chosen": -120.32958984375, | |
| "logps/rejected": -101.76454162597656, | |
| "loss": 0.331, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 1.0935906171798706, | |
| "rewards/margins": 1.1072776317596436, | |
| "rewards/rejected": -0.013687020167708397, | |
| "step": 288 | |
| }, | |
| { | |
| "epoch": 18.54, | |
| "grad_norm": 5.824638843536377, | |
| "learning_rate": 4.884528244389268e-07, | |
| "logits/chosen": -2.6889617443084717, | |
| "logits/rejected": -2.690692663192749, | |
| "logps/chosen": -120.37040710449219, | |
| "logps/rejected": -98.27632141113281, | |
| "loss": 0.3204, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 1.1049582958221436, | |
| "rewards/margins": 1.175376057624817, | |
| "rewards/rejected": -0.07041767239570618, | |
| "step": 292 | |
| }, | |
| { | |
| "epoch": 18.79, | |
| "grad_norm": 5.431331634521484, | |
| "learning_rate": 4.881109094011253e-07, | |
| "logits/chosen": -2.740906238555908, | |
| "logits/rejected": -2.725914478302002, | |
| "logps/chosen": -124.35327911376953, | |
| "logps/rejected": -100.71537780761719, | |
| "loss": 0.2884, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.2057325839996338, | |
| "rewards/margins": 1.3113868236541748, | |
| "rewards/rejected": -0.10565409809350967, | |
| "step": 296 | |
| }, | |
| { | |
| "epoch": 19.05, | |
| "grad_norm": 4.33558988571167, | |
| "learning_rate": 4.877641290737883e-07, | |
| "logits/chosen": -2.700420379638672, | |
| "logits/rejected": -2.7527971267700195, | |
| "logps/chosen": -117.01264953613281, | |
| "logps/rejected": -101.50834655761719, | |
| "loss": 0.2757, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.150115728378296, | |
| "rewards/margins": 1.34402596950531, | |
| "rewards/rejected": -0.19391010701656342, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 19.3, | |
| "grad_norm": 4.987298488616943, | |
| "learning_rate": 4.874124905426338e-07, | |
| "logits/chosen": -2.715477705001831, | |
| "logits/rejected": -2.760146141052246, | |
| "logps/chosen": -117.20336151123047, | |
| "logps/rejected": -94.89369201660156, | |
| "loss": 0.3244, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 1.107708215713501, | |
| "rewards/margins": 1.1569105386734009, | |
| "rewards/rejected": -0.04920227825641632, | |
| "step": 304 | |
| }, | |
| { | |
| "epoch": 19.56, | |
| "grad_norm": 4.090747833251953, | |
| "learning_rate": 4.870560009926464e-07, | |
| "logits/chosen": -2.6785225868225098, | |
| "logits/rejected": -2.6984167098999023, | |
| "logps/chosen": -112.27914428710938, | |
| "logps/rejected": -94.59768676757812, | |
| "loss": 0.2833, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.2028326988220215, | |
| "rewards/margins": 1.334948182106018, | |
| "rewards/rejected": -0.13211557269096375, | |
| "step": 308 | |
| }, | |
| { | |
| "epoch": 19.81, | |
| "grad_norm": 5.0767083168029785, | |
| "learning_rate": 4.866946677079314e-07, | |
| "logits/chosen": -2.712416410446167, | |
| "logits/rejected": -2.7296035289764404, | |
| "logps/chosen": -121.65861511230469, | |
| "logps/rejected": -106.11611938476562, | |
| "loss": 0.2893, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.1842504739761353, | |
| "rewards/margins": 1.3191744089126587, | |
| "rewards/rejected": -0.13492396473884583, | |
| "step": 312 | |
| }, | |
| { | |
| "epoch": 20.06, | |
| "grad_norm": 4.1403326988220215, | |
| "learning_rate": 4.863284980715648e-07, | |
| "logits/chosen": -2.658827781677246, | |
| "logits/rejected": -2.631199836730957, | |
| "logps/chosen": -129.3588409423828, | |
| "logps/rejected": -103.25465393066406, | |
| "loss": 0.2678, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.291969895362854, | |
| "rewards/margins": 1.4036282300949097, | |
| "rewards/rejected": -0.11165820807218552, | |
| "step": 316 | |
| }, | |
| { | |
| "epoch": 20.32, | |
| "grad_norm": 4.350497245788574, | |
| "learning_rate": 4.859574995654441e-07, | |
| "logits/chosen": -2.761063814163208, | |
| "logits/rejected": -2.7751922607421875, | |
| "logps/chosen": -114.76087951660156, | |
| "logps/rejected": -100.04124450683594, | |
| "loss": 0.298, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.1642746925354004, | |
| "rewards/margins": 1.3020979166030884, | |
| "rewards/rejected": -0.13782335817813873, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 20.32, | |
| "eval_logits/chosen": -2.792503595352173, | |
| "eval_logits/rejected": -2.870570421218872, | |
| "eval_logps/chosen": -123.9360122680664, | |
| "eval_logps/rejected": -96.95860290527344, | |
| "eval_loss": 0.4256473481655121, | |
| "eval_rewards/accuracies": 0.8214285969734192, | |
| "eval_rewards/chosen": 1.0133329629898071, | |
| "eval_rewards/margins": 0.8983038663864136, | |
| "eval_rewards/rejected": 0.11502902954816818, | |
| "eval_runtime": 19.3701, | |
| "eval_samples_per_second": 1.446, | |
| "eval_steps_per_second": 1.446, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 20.57, | |
| "grad_norm": 4.334249973297119, | |
| "learning_rate": 4.855816797701336e-07, | |
| "logits/chosen": -2.690117359161377, | |
| "logits/rejected": -2.7377476692199707, | |
| "logps/chosen": -116.01565551757812, | |
| "logps/rejected": -102.1231689453125, | |
| "loss": 0.2872, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 1.2394335269927979, | |
| "rewards/margins": 1.3586478233337402, | |
| "rewards/rejected": -0.11921422928571701, | |
| "step": 324 | |
| }, | |
| { | |
| "epoch": 20.83, | |
| "grad_norm": 4.597221851348877, | |
| "learning_rate": 4.85201046364711e-07, | |
| "logits/chosen": -2.741600751876831, | |
| "logits/rejected": -2.712954044342041, | |
| "logps/chosen": -124.32752990722656, | |
| "logps/rejected": -101.2073974609375, | |
| "loss": 0.2676, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.22231924533844, | |
| "rewards/margins": 1.4293614625930786, | |
| "rewards/rejected": -0.20704227685928345, | |
| "step": 328 | |
| }, | |
| { | |
| "epoch": 21.08, | |
| "grad_norm": 3.682258367538452, | |
| "learning_rate": 4.848156071266095e-07, | |
| "logits/chosen": -2.6585566997528076, | |
| "logits/rejected": -2.704587459564209, | |
| "logps/chosen": -120.91087341308594, | |
| "logps/rejected": -98.83493041992188, | |
| "loss": 0.2705, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.2277172803878784, | |
| "rewards/margins": 1.3790152072906494, | |
| "rewards/rejected": -0.15129785239696503, | |
| "step": 332 | |
| }, | |
| { | |
| "epoch": 21.33, | |
| "grad_norm": 3.7775039672851562, | |
| "learning_rate": 4.844253699314596e-07, | |
| "logits/chosen": -2.7104837894439697, | |
| "logits/rejected": -2.7378451824188232, | |
| "logps/chosen": -117.22598266601562, | |
| "logps/rejected": -97.59563446044922, | |
| "loss": 0.2755, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.2548604011535645, | |
| "rewards/margins": 1.3694878816604614, | |
| "rewards/rejected": -0.11462761461734772, | |
| "step": 336 | |
| }, | |
| { | |
| "epoch": 21.59, | |
| "grad_norm": 4.84611701965332, | |
| "learning_rate": 4.840303427529273e-07, | |
| "logits/chosen": -2.6837968826293945, | |
| "logits/rejected": -2.7071595191955566, | |
| "logps/chosen": -121.88581085205078, | |
| "logps/rejected": -102.45454406738281, | |
| "loss": 0.286, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 1.2159500122070312, | |
| "rewards/margins": 1.3315218687057495, | |
| "rewards/rejected": -0.11557181179523468, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 21.84, | |
| "grad_norm": 3.760068655014038, | |
| "learning_rate": 4.836305336625523e-07, | |
| "logits/chosen": -2.709066390991211, | |
| "logits/rejected": -2.73175048828125, | |
| "logps/chosen": -117.1901626586914, | |
| "logps/rejected": -105.62377166748047, | |
| "loss": 0.2661, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.3225661516189575, | |
| "rewards/margins": 1.445369005203247, | |
| "rewards/rejected": -0.12280277907848358, | |
| "step": 344 | |
| }, | |
| { | |
| "epoch": 22.1, | |
| "grad_norm": 3.8488616943359375, | |
| "learning_rate": 4.832259508295822e-07, | |
| "logits/chosen": -2.7598538398742676, | |
| "logits/rejected": -2.776313066482544, | |
| "logps/chosen": -117.15644836425781, | |
| "logps/rejected": -93.95286560058594, | |
| "loss": 0.2711, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 1.3003535270690918, | |
| "rewards/margins": 1.5021978616714478, | |
| "rewards/rejected": -0.20184442400932312, | |
| "step": 348 | |
| }, | |
| { | |
| "epoch": 22.35, | |
| "grad_norm": 3.152104377746582, | |
| "learning_rate": 4.828166025208058e-07, | |
| "logits/chosen": -2.6503114700317383, | |
| "logits/rejected": -2.6512064933776855, | |
| "logps/chosen": -118.29179382324219, | |
| "logps/rejected": -97.83879852294922, | |
| "loss": 0.2394, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.2750773429870605, | |
| "rewards/margins": 1.585729718208313, | |
| "rewards/rejected": -0.31065240502357483, | |
| "step": 352 | |
| }, | |
| { | |
| "epoch": 22.6, | |
| "grad_norm": 4.1737165451049805, | |
| "learning_rate": 4.824024971003845e-07, | |
| "logits/chosen": -2.708979845046997, | |
| "logits/rejected": -2.718264579772949, | |
| "logps/chosen": -113.75922393798828, | |
| "logps/rejected": -100.5313720703125, | |
| "loss": 0.2626, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.3227206468582153, | |
| "rewards/margins": 1.4637939929962158, | |
| "rewards/rejected": -0.1410733461380005, | |
| "step": 356 | |
| }, | |
| { | |
| "epoch": 22.86, | |
| "grad_norm": 4.2186737060546875, | |
| "learning_rate": 4.819836430296808e-07, | |
| "logits/chosen": -2.688643217086792, | |
| "logits/rejected": -2.7126376628875732, | |
| "logps/chosen": -119.3924560546875, | |
| "logps/rejected": -102.88687896728516, | |
| "loss": 0.272, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.3319997787475586, | |
| "rewards/margins": 1.3775808811187744, | |
| "rewards/rejected": -0.04558102786540985, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 22.86, | |
| "eval_logits/chosen": -2.7863781452178955, | |
| "eval_logits/rejected": -2.8632652759552, | |
| "eval_logps/chosen": -123.22322082519531, | |
| "eval_logps/rejected": -97.17105865478516, | |
| "eval_loss": 0.4147447645664215, | |
| "eval_rewards/accuracies": 0.8214285969734192, | |
| "eval_rewards/chosen": 1.0846121311187744, | |
| "eval_rewards/margins": 0.9908290505409241, | |
| "eval_rewards/rejected": 0.09378316253423691, | |
| "eval_runtime": 19.3953, | |
| "eval_samples_per_second": 1.444, | |
| "eval_steps_per_second": 1.444, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 23.11, | |
| "grad_norm": 3.8801398277282715, | |
| "learning_rate": 4.815600488670862e-07, | |
| "logits/chosen": -2.661937713623047, | |
| "logits/rejected": -2.69406795501709, | |
| "logps/chosen": -120.42941284179688, | |
| "logps/rejected": -107.56249237060547, | |
| "loss": 0.2463, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.303568959236145, | |
| "rewards/margins": 1.5796161890029907, | |
| "rewards/rejected": -0.2760472595691681, | |
| "step": 364 | |
| }, | |
| { | |
| "epoch": 23.37, | |
| "grad_norm": 3.6953530311584473, | |
| "learning_rate": 4.811317232678456e-07, | |
| "logits/chosen": -2.696389675140381, | |
| "logits/rejected": -2.7162017822265625, | |
| "logps/chosen": -115.95695495605469, | |
| "logps/rejected": -95.26593017578125, | |
| "loss": 0.2557, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.3928656578063965, | |
| "rewards/margins": 1.5144150257110596, | |
| "rewards/rejected": -0.12154923379421234, | |
| "step": 368 | |
| }, | |
| { | |
| "epoch": 23.62, | |
| "grad_norm": 4.2311553955078125, | |
| "learning_rate": 4.806986749838806e-07, | |
| "logits/chosen": -2.735870599746704, | |
| "logits/rejected": -2.7359132766723633, | |
| "logps/chosen": -120.84663391113281, | |
| "logps/rejected": -107.17831420898438, | |
| "loss": 0.2922, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.2136850357055664, | |
| "rewards/margins": 1.3251327276229858, | |
| "rewards/rejected": -0.11144763976335526, | |
| "step": 372 | |
| }, | |
| { | |
| "epoch": 23.87, | |
| "grad_norm": 3.250326633453369, | |
| "learning_rate": 4.802609128636112e-07, | |
| "logits/chosen": -2.6809797286987305, | |
| "logits/rejected": -2.700901508331299, | |
| "logps/chosen": -121.92070007324219, | |
| "logps/rejected": -99.91931915283203, | |
| "loss": 0.2062, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.4374094009399414, | |
| "rewards/margins": 1.8276853561401367, | |
| "rewards/rejected": -0.3902759850025177, | |
| "step": 376 | |
| }, | |
| { | |
| "epoch": 24.13, | |
| "grad_norm": 4.59958553314209, | |
| "learning_rate": 4.798184458517744e-07, | |
| "logits/chosen": -2.7332396507263184, | |
| "logits/rejected": -2.768554210662842, | |
| "logps/chosen": -114.1657943725586, | |
| "logps/rejected": -100.7848892211914, | |
| "loss": 0.2353, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.4016469717025757, | |
| "rewards/margins": 1.5988764762878418, | |
| "rewards/rejected": -0.19722968339920044, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 24.38, | |
| "grad_norm": 3.6598093509674072, | |
| "learning_rate": 4.793712829892414e-07, | |
| "logits/chosen": -2.7024924755096436, | |
| "logits/rejected": -2.7085440158843994, | |
| "logps/chosen": -114.50930786132812, | |
| "logps/rejected": -104.57618713378906, | |
| "loss": 0.2538, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.4238942861557007, | |
| "rewards/margins": 1.5259473323822021, | |
| "rewards/rejected": -0.10205309092998505, | |
| "step": 384 | |
| }, | |
| { | |
| "epoch": 24.63, | |
| "grad_norm": 3.3857998847961426, | |
| "learning_rate": 4.789194334128338e-07, | |
| "logits/chosen": -2.7139151096343994, | |
| "logits/rejected": -2.7087197303771973, | |
| "logps/chosen": -121.57815551757812, | |
| "logps/rejected": -102.66044616699219, | |
| "loss": 0.2324, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.3619202375411987, | |
| "rewards/margins": 1.667473316192627, | |
| "rewards/rejected": -0.305553138256073, | |
| "step": 388 | |
| }, | |
| { | |
| "epoch": 24.89, | |
| "grad_norm": 4.038039207458496, | |
| "learning_rate": 4.784629063551354e-07, | |
| "logits/chosen": -2.651282548904419, | |
| "logits/rejected": -2.679689884185791, | |
| "logps/chosen": -118.9875717163086, | |
| "logps/rejected": -97.62678527832031, | |
| "loss": 0.2304, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 1.3316906690597534, | |
| "rewards/margins": 1.7062404155731201, | |
| "rewards/rejected": -0.3745496869087219, | |
| "step": 392 | |
| }, | |
| { | |
| "epoch": 25.14, | |
| "grad_norm": 3.0911149978637695, | |
| "learning_rate": 4.780017111443047e-07, | |
| "logits/chosen": -2.718074083328247, | |
| "logits/rejected": -2.7447762489318848, | |
| "logps/chosen": -121.20558166503906, | |
| "logps/rejected": -105.63926696777344, | |
| "loss": 0.2411, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.4981815814971924, | |
| "rewards/margins": 1.644660472869873, | |
| "rewards/rejected": -0.14647886157035828, | |
| "step": 396 | |
| }, | |
| { | |
| "epoch": 25.4, | |
| "grad_norm": 3.236288070678711, | |
| "learning_rate": 4.775358572038845e-07, | |
| "logits/chosen": -2.693341016769409, | |
| "logits/rejected": -2.681511878967285, | |
| "logps/chosen": -114.28794860839844, | |
| "logps/rejected": -91.69850158691406, | |
| "loss": 0.259, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.3284053802490234, | |
| "rewards/margins": 1.5037521123886108, | |
| "rewards/rejected": -0.17534679174423218, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 25.4, | |
| "eval_logits/chosen": -2.7796428203582764, | |
| "eval_logits/rejected": -2.8552372455596924, | |
| "eval_logps/chosen": -122.60997009277344, | |
| "eval_logps/rejected": -97.46685028076172, | |
| "eval_loss": 0.4054030478000641, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.1459368467330933, | |
| "eval_rewards/margins": 1.0817322731018066, | |
| "eval_rewards/rejected": 0.06420455127954483, | |
| "eval_runtime": 19.3983, | |
| "eval_samples_per_second": 1.443, | |
| "eval_steps_per_second": 1.443, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 25.65, | |
| "grad_norm": 3.2894155979156494, | |
| "learning_rate": 4.770653540526078e-07, | |
| "logits/chosen": -2.652012825012207, | |
| "logits/rejected": -2.690704345703125, | |
| "logps/chosen": -118.52864837646484, | |
| "logps/rejected": -102.0433578491211, | |
| "loss": 0.1918, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.5184407234191895, | |
| "rewards/margins": 1.8730953931808472, | |
| "rewards/rejected": -0.3546547293663025, | |
| "step": 404 | |
| }, | |
| { | |
| "epoch": 25.9, | |
| "grad_norm": 4.958095550537109, | |
| "learning_rate": 4.765902113042053e-07, | |
| "logits/chosen": -2.7191715240478516, | |
| "logits/rejected": -2.701702833175659, | |
| "logps/chosen": -115.93360900878906, | |
| "logps/rejected": -100.49573516845703, | |
| "loss": 0.2536, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.357507348060608, | |
| "rewards/margins": 1.542129635810852, | |
| "rewards/rejected": -0.18462230265140533, | |
| "step": 408 | |
| }, | |
| { | |
| "epoch": 26.16, | |
| "grad_norm": 3.1129214763641357, | |
| "learning_rate": 4.7611043866720737e-07, | |
| "logits/chosen": -2.6934893131256104, | |
| "logits/rejected": -2.7433109283447266, | |
| "logps/chosen": -117.78741455078125, | |
| "logps/rejected": -102.46746063232422, | |
| "loss": 0.2137, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 1.484704613685608, | |
| "rewards/margins": 1.8401193618774414, | |
| "rewards/rejected": -0.3554147481918335, | |
| "step": 412 | |
| }, | |
| { | |
| "epoch": 26.41, | |
| "grad_norm": 3.2823047637939453, | |
| "learning_rate": 4.7562604594474643e-07, | |
| "logits/chosen": -2.6928577423095703, | |
| "logits/rejected": -2.708937644958496, | |
| "logps/chosen": -124.94667053222656, | |
| "logps/rejected": -106.61329650878906, | |
| "loss": 0.2137, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.471519112586975, | |
| "rewards/margins": 1.7852158546447754, | |
| "rewards/rejected": -0.31369656324386597, | |
| "step": 416 | |
| }, | |
| { | |
| "epoch": 26.67, | |
| "grad_norm": 3.7071120738983154, | |
| "learning_rate": 4.7513704303435674e-07, | |
| "logits/chosen": -2.670248031616211, | |
| "logits/rejected": -2.6764914989471436, | |
| "logps/chosen": -110.71532440185547, | |
| "logps/rejected": -101.84510803222656, | |
| "loss": 0.229, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.4525586366653442, | |
| "rewards/margins": 1.6690635681152344, | |
| "rewards/rejected": -0.21650481224060059, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 26.92, | |
| "grad_norm": 3.449319362640381, | |
| "learning_rate": 4.7464343992777165e-07, | |
| "logits/chosen": -2.688829183578491, | |
| "logits/rejected": -2.7286489009857178, | |
| "logps/chosen": -115.69140625, | |
| "logps/rejected": -103.97221374511719, | |
| "loss": 0.2231, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.4910831451416016, | |
| "rewards/margins": 1.7472798824310303, | |
| "rewards/rejected": -0.2561967670917511, | |
| "step": 424 | |
| }, | |
| { | |
| "epoch": 27.17, | |
| "grad_norm": 2.4958574771881104, | |
| "learning_rate": 4.741452467107199e-07, | |
| "logits/chosen": -2.6915664672851562, | |
| "logits/rejected": -2.7021422386169434, | |
| "logps/chosen": -110.91499328613281, | |
| "logps/rejected": -98.70269775390625, | |
| "loss": 0.233, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.4354866743087769, | |
| "rewards/margins": 1.7168078422546387, | |
| "rewards/rejected": -0.28132107853889465, | |
| "step": 428 | |
| }, | |
| { | |
| "epoch": 27.43, | |
| "grad_norm": 3.7154319286346436, | |
| "learning_rate": 4.7364247356271927e-07, | |
| "logits/chosen": -2.6695199012756348, | |
| "logits/rejected": -2.6698789596557617, | |
| "logps/chosen": -118.46913146972656, | |
| "logps/rejected": -102.86263275146484, | |
| "loss": 0.2337, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.4410640001296997, | |
| "rewards/margins": 1.6770535707473755, | |
| "rewards/rejected": -0.23598948121070862, | |
| "step": 432 | |
| }, | |
| { | |
| "epoch": 27.68, | |
| "grad_norm": 3.101527690887451, | |
| "learning_rate": 4.731351307568687e-07, | |
| "logits/chosen": -2.6910083293914795, | |
| "logits/rejected": -2.6801414489746094, | |
| "logps/chosen": -115.44275665283203, | |
| "logps/rejected": -97.7877197265625, | |
| "loss": 0.2, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.5193870067596436, | |
| "rewards/margins": 1.8750090599060059, | |
| "rewards/rejected": -0.35562214255332947, | |
| "step": 436 | |
| }, | |
| { | |
| "epoch": 27.94, | |
| "grad_norm": 3.9017672538757324, | |
| "learning_rate": 4.7262322865963846e-07, | |
| "logits/chosen": -2.7122743129730225, | |
| "logits/rejected": -2.723294258117676, | |
| "logps/chosen": -116.90724182128906, | |
| "logps/rejected": -104.45236206054688, | |
| "loss": 0.1959, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.5187357664108276, | |
| "rewards/margins": 1.918578863143921, | |
| "rewards/rejected": -0.39984312653541565, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 27.94, | |
| "eval_logits/chosen": -2.7724335193634033, | |
| "eval_logits/rejected": -2.8468399047851562, | |
| "eval_logps/chosen": -122.11509704589844, | |
| "eval_logps/rejected": -97.87715148925781, | |
| "eval_loss": 0.39711207151412964, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.1954243183135986, | |
| "eval_rewards/margins": 1.1722500324249268, | |
| "eval_rewards/rejected": 0.023174213245511055, | |
| "eval_runtime": 19.4446, | |
| "eval_samples_per_second": 1.44, | |
| "eval_steps_per_second": 1.44, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 28.19, | |
| "grad_norm": 4.0338592529296875, | |
| "learning_rate": 4.721067777306581e-07, | |
| "logits/chosen": -2.6722593307495117, | |
| "logits/rejected": -2.682727575302124, | |
| "logps/chosen": -119.6982650756836, | |
| "logps/rejected": -105.52352905273438, | |
| "loss": 0.2108, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.5048611164093018, | |
| "rewards/margins": 1.8753631114959717, | |
| "rewards/rejected": -0.3705020546913147, | |
| "step": 444 | |
| }, | |
| { | |
| "epoch": 28.44, | |
| "grad_norm": 3.167048692703247, | |
| "learning_rate": 4.71585788522503e-07, | |
| "logits/chosen": -2.652644157409668, | |
| "logits/rejected": -2.6818065643310547, | |
| "logps/chosen": -113.42478942871094, | |
| "logps/rejected": -104.37007141113281, | |
| "loss": 0.2076, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.474198341369629, | |
| "rewards/margins": 1.7973999977111816, | |
| "rewards/rejected": -0.32320165634155273, | |
| "step": 448 | |
| }, | |
| { | |
| "epoch": 28.7, | |
| "grad_norm": 3.4248368740081787, | |
| "learning_rate": 4.7106027168047833e-07, | |
| "logits/chosen": -2.7117884159088135, | |
| "logits/rejected": -2.77234148979187, | |
| "logps/chosen": -116.26769256591797, | |
| "logps/rejected": -101.22008514404297, | |
| "loss": 0.1858, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.5492609739303589, | |
| "rewards/margins": 1.9960644245147705, | |
| "rewards/rejected": -0.4468035101890564, | |
| "step": 452 | |
| }, | |
| { | |
| "epoch": 28.95, | |
| "grad_norm": 2.855328321456909, | |
| "learning_rate": 4.705302379424022e-07, | |
| "logits/chosen": -2.6717069149017334, | |
| "logits/rejected": -2.672755718231201, | |
| "logps/chosen": -116.75204467773438, | |
| "logps/rejected": -106.22608184814453, | |
| "loss": 0.2065, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.5143406391143799, | |
| "rewards/margins": 1.8520642518997192, | |
| "rewards/rejected": -0.33772367238998413, | |
| "step": 456 | |
| }, | |
| { | |
| "epoch": 29.21, | |
| "grad_norm": 2.8747594356536865, | |
| "learning_rate": 4.699956981383857e-07, | |
| "logits/chosen": -2.693591356277466, | |
| "logits/rejected": -2.662400960922241, | |
| "logps/chosen": -120.1160888671875, | |
| "logps/rejected": -91.72486877441406, | |
| "loss": 0.1874, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.6478428840637207, | |
| "rewards/margins": 2.0628483295440674, | |
| "rewards/rejected": -0.4150054454803467, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 29.46, | |
| "grad_norm": 2.8170723915100098, | |
| "learning_rate": 4.694566631906116e-07, | |
| "logits/chosen": -2.6773173809051514, | |
| "logits/rejected": -2.672825574874878, | |
| "logps/chosen": -113.59075927734375, | |
| "logps/rejected": -103.25709533691406, | |
| "loss": 0.1984, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.4863029718399048, | |
| "rewards/margins": 1.892244577407837, | |
| "rewards/rejected": -0.405941367149353, | |
| "step": 464 | |
| }, | |
| { | |
| "epoch": 29.71, | |
| "grad_norm": 3.132080078125, | |
| "learning_rate": 4.689131441131118e-07, | |
| "logits/chosen": -2.6520447731018066, | |
| "logits/rejected": -2.6872832775115967, | |
| "logps/chosen": -118.07131958007812, | |
| "logps/rejected": -107.88346862792969, | |
| "loss": 0.2029, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.6008590459823608, | |
| "rewards/margins": 1.909461259841919, | |
| "rewards/rejected": -0.30860215425491333, | |
| "step": 468 | |
| }, | |
| { | |
| "epoch": 29.97, | |
| "grad_norm": 3.000974416732788, | |
| "learning_rate": 4.6836515201154135e-07, | |
| "logits/chosen": -2.709559202194214, | |
| "logits/rejected": -2.723712205886841, | |
| "logps/chosen": -112.09906768798828, | |
| "logps/rejected": -106.09660339355469, | |
| "loss": 0.2037, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.5191339254379272, | |
| "rewards/margins": 1.8524271249771118, | |
| "rewards/rejected": -0.3332933187484741, | |
| "step": 472 | |
| }, | |
| { | |
| "epoch": 30.22, | |
| "grad_norm": 2.544147491455078, | |
| "learning_rate": 4.678126980829524e-07, | |
| "logits/chosen": -2.678325653076172, | |
| "logits/rejected": -2.699833631515503, | |
| "logps/chosen": -115.4670639038086, | |
| "logps/rejected": -101.5027847290039, | |
| "loss": 0.1813, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.4956222772598267, | |
| "rewards/margins": 1.9856685400009155, | |
| "rewards/rejected": -0.4900462329387665, | |
| "step": 476 | |
| }, | |
| { | |
| "epoch": 30.48, | |
| "grad_norm": 2.9770796298980713, | |
| "learning_rate": 4.672557936155649e-07, | |
| "logits/chosen": -2.708721160888672, | |
| "logits/rejected": -2.728306770324707, | |
| "logps/chosen": -109.71602630615234, | |
| "logps/rejected": -98.1273422241211, | |
| "loss": 0.1946, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.5404164791107178, | |
| "rewards/margins": 1.9511140584945679, | |
| "rewards/rejected": -0.4106977581977844, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 30.48, | |
| "eval_logits/chosen": -2.764019012451172, | |
| "eval_logits/rejected": -2.8370797634124756, | |
| "eval_logps/chosen": -121.74720001220703, | |
| "eval_logps/rejected": -98.30638122558594, | |
| "eval_loss": 0.393496572971344, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.232213020324707, | |
| "eval_rewards/margins": 1.2519627809524536, | |
| "eval_rewards/rejected": -0.019749613478779793, | |
| "eval_runtime": 19.4183, | |
| "eval_samples_per_second": 1.442, | |
| "eval_steps_per_second": 1.442, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 30.73, | |
| "grad_norm": 3.826716661453247, | |
| "learning_rate": 4.6669444998853605e-07, | |
| "logits/chosen": -2.6732892990112305, | |
| "logits/rejected": -2.700220823287964, | |
| "logps/chosen": -117.65553283691406, | |
| "logps/rejected": -111.19263458251953, | |
| "loss": 0.1876, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.6948100328445435, | |
| "rewards/margins": 2.0582737922668457, | |
| "rewards/rejected": -0.3634639382362366, | |
| "step": 484 | |
| }, | |
| { | |
| "epoch": 30.98, | |
| "grad_norm": 3.326704740524292, | |
| "learning_rate": 4.661286786717278e-07, | |
| "logits/chosen": -2.6515111923217773, | |
| "logits/rejected": -2.6504664421081543, | |
| "logps/chosen": -120.06623840332031, | |
| "logps/rejected": -101.47470092773438, | |
| "loss": 0.1828, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.5970648527145386, | |
| "rewards/margins": 2.0543787479400635, | |
| "rewards/rejected": -0.45731377601623535, | |
| "step": 488 | |
| }, | |
| { | |
| "epoch": 31.24, | |
| "grad_norm": 2.700209856033325, | |
| "learning_rate": 4.6555849122547263e-07, | |
| "logits/chosen": -2.6469173431396484, | |
| "logits/rejected": -2.666424036026001, | |
| "logps/chosen": -111.76505279541016, | |
| "logps/rejected": -103.40081024169922, | |
| "loss": 0.185, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.524863839149475, | |
| "rewards/margins": 1.9786373376846313, | |
| "rewards/rejected": -0.4537736773490906, | |
| "step": 492 | |
| }, | |
| { | |
| "epoch": 31.49, | |
| "grad_norm": 2.592606544494629, | |
| "learning_rate": 4.649838993003372e-07, | |
| "logits/chosen": -2.7192447185516357, | |
| "logits/rejected": -2.726841926574707, | |
| "logps/chosen": -112.72746276855469, | |
| "logps/rejected": -100.51376342773438, | |
| "loss": 0.1678, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.5847443342208862, | |
| "rewards/margins": 2.0699594020843506, | |
| "rewards/rejected": -0.4852150082588196, | |
| "step": 496 | |
| }, | |
| { | |
| "epoch": 31.75, | |
| "grad_norm": 2.768076181411743, | |
| "learning_rate": 4.644049146368843e-07, | |
| "logits/chosen": -2.6624693870544434, | |
| "logits/rejected": -2.677476406097412, | |
| "logps/chosen": -119.13687896728516, | |
| "logps/rejected": -103.49778747558594, | |
| "loss": 0.1873, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7677032947540283, | |
| "rewards/margins": 2.089017868041992, | |
| "rewards/rejected": -0.32131490111351013, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 32.0, | |
| "grad_norm": 2.4647607803344727, | |
| "learning_rate": 4.638215490654329e-07, | |
| "logits/chosen": -2.6946330070495605, | |
| "logits/rejected": -2.6913187503814697, | |
| "logps/chosen": -118.64775085449219, | |
| "logps/rejected": -107.40900421142578, | |
| "loss": 0.1847, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.5691322088241577, | |
| "rewards/margins": 2.105942964553833, | |
| "rewards/rejected": -0.5368106365203857, | |
| "step": 504 | |
| }, | |
| { | |
| "epoch": 32.25, | |
| "grad_norm": 3.723209857940674, | |
| "learning_rate": 4.6323381450581665e-07, | |
| "logits/chosen": -2.6984715461730957, | |
| "logits/rejected": -2.686072826385498, | |
| "logps/chosen": -118.19576263427734, | |
| "logps/rejected": -104.87760162353516, | |
| "loss": 0.1853, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.504220724105835, | |
| "rewards/margins": 2.023249864578247, | |
| "rewards/rejected": -0.519028902053833, | |
| "step": 508 | |
| }, | |
| { | |
| "epoch": 32.51, | |
| "grad_norm": 2.6810081005096436, | |
| "learning_rate": 4.6264172296714e-07, | |
| "logits/chosen": -2.6358213424682617, | |
| "logits/rejected": -2.6452157497406006, | |
| "logps/chosen": -116.47501373291016, | |
| "logps/rejected": -104.9029769897461, | |
| "loss": 0.1948, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.6027023792266846, | |
| "rewards/margins": 2.105417251586914, | |
| "rewards/rejected": -0.5027150511741638, | |
| "step": 512 | |
| }, | |
| { | |
| "epoch": 32.76, | |
| "grad_norm": 2.1455435752868652, | |
| "learning_rate": 4.6204528654753304e-07, | |
| "logits/chosen": -2.692194700241089, | |
| "logits/rejected": -2.705289125442505, | |
| "logps/chosen": -113.38731384277344, | |
| "logps/rejected": -103.49237823486328, | |
| "loss": 0.1617, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7136329412460327, | |
| "rewards/margins": 2.1900761127471924, | |
| "rewards/rejected": -0.47644317150115967, | |
| "step": 516 | |
| }, | |
| { | |
| "epoch": 33.02, | |
| "grad_norm": 3.499128818511963, | |
| "learning_rate": 4.614445174339045e-07, | |
| "logits/chosen": -2.661409616470337, | |
| "logits/rejected": -2.700990915298462, | |
| "logps/chosen": -113.71514892578125, | |
| "logps/rejected": -103.91769409179688, | |
| "loss": 0.1507, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7427312135696411, | |
| "rewards/margins": 2.1958351135253906, | |
| "rewards/rejected": -0.45310381054878235, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 33.02, | |
| "eval_logits/chosen": -2.755352020263672, | |
| "eval_logits/rejected": -2.826873540878296, | |
| "eval_logps/chosen": -121.47989654541016, | |
| "eval_logps/rejected": -98.8564453125, | |
| "eval_loss": 0.3899853527545929, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.2589442729949951, | |
| "eval_rewards/margins": 1.333700180053711, | |
| "eval_rewards/rejected": -0.07475597411394119, | |
| "eval_runtime": 19.3899, | |
| "eval_samples_per_second": 1.444, | |
| "eval_steps_per_second": 1.444, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 33.27, | |
| "grad_norm": 2.419506549835205, | |
| "learning_rate": 4.6083942790169206e-07, | |
| "logits/chosen": -2.6735386848449707, | |
| "logits/rejected": -2.696384906768799, | |
| "logps/chosen": -111.79097747802734, | |
| "logps/rejected": -98.7368392944336, | |
| "loss": 0.1581, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.5749679803848267, | |
| "rewards/margins": 2.265718460083008, | |
| "rewards/rejected": -0.690750777721405, | |
| "step": 524 | |
| }, | |
| { | |
| "epoch": 33.52, | |
| "grad_norm": 2.1046078205108643, | |
| "learning_rate": 4.6023003031461227e-07, | |
| "logits/chosen": -2.6465351581573486, | |
| "logits/rejected": -2.6622884273529053, | |
| "logps/chosen": -114.73282623291016, | |
| "logps/rejected": -103.12973022460938, | |
| "loss": 0.1778, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.6710339784622192, | |
| "rewards/margins": 2.119300365447998, | |
| "rewards/rejected": -0.4482663571834564, | |
| "step": 528 | |
| }, | |
| { | |
| "epoch": 33.78, | |
| "grad_norm": 3.0457003116607666, | |
| "learning_rate": 4.596163371244076e-07, | |
| "logits/chosen": -2.6422927379608154, | |
| "logits/rejected": -2.6462604999542236, | |
| "logps/chosen": -114.708740234375, | |
| "logps/rejected": -105.07784271240234, | |
| "loss": 0.16, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7936890125274658, | |
| "rewards/margins": 2.2034826278686523, | |
| "rewards/rejected": -0.40979355573654175, | |
| "step": 532 | |
| }, | |
| { | |
| "epoch": 34.03, | |
| "grad_norm": 2.8257133960723877, | |
| "learning_rate": 4.5899836087059174e-07, | |
| "logits/chosen": -2.6918063163757324, | |
| "logits/rejected": -2.696005344390869, | |
| "logps/chosen": -117.54481506347656, | |
| "logps/rejected": -111.74267578125, | |
| "loss": 0.1753, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.625847339630127, | |
| "rewards/margins": 2.2450764179229736, | |
| "rewards/rejected": -0.6192291975021362, | |
| "step": 536 | |
| }, | |
| { | |
| "epoch": 34.29, | |
| "grad_norm": 2.5381155014038086, | |
| "learning_rate": 4.58376114180194e-07, | |
| "logits/chosen": -2.684847116470337, | |
| "logits/rejected": -2.6875247955322266, | |
| "logps/chosen": -121.1004867553711, | |
| "logps/rejected": -101.69570922851562, | |
| "loss": 0.1562, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.6668047904968262, | |
| "rewards/margins": 2.3632285594940186, | |
| "rewards/rejected": -0.6964237689971924, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 34.54, | |
| "grad_norm": 2.519352912902832, | |
| "learning_rate": 4.577496097675009e-07, | |
| "logits/chosen": -2.6664843559265137, | |
| "logits/rejected": -2.681034803390503, | |
| "logps/chosen": -115.94261169433594, | |
| "logps/rejected": -108.1679458618164, | |
| "loss": 0.1516, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7702218294143677, | |
| "rewards/margins": 2.2215099334716797, | |
| "rewards/rejected": -0.4512881338596344, | |
| "step": 544 | |
| }, | |
| { | |
| "epoch": 34.79, | |
| "grad_norm": 2.789398431777954, | |
| "learning_rate": 4.571188604337962e-07, | |
| "logits/chosen": -2.668377637863159, | |
| "logits/rejected": -2.688797950744629, | |
| "logps/chosen": -108.2844467163086, | |
| "logps/rejected": -104.94053649902344, | |
| "loss": 0.1838, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.559061050415039, | |
| "rewards/margins": 2.064011812210083, | |
| "rewards/rejected": -0.504950761795044, | |
| "step": 548 | |
| }, | |
| { | |
| "epoch": 35.05, | |
| "grad_norm": 3.570829153060913, | |
| "learning_rate": 4.5648387906709995e-07, | |
| "logits/chosen": -2.6559903621673584, | |
| "logits/rejected": -2.650508165359497, | |
| "logps/chosen": -112.94985961914062, | |
| "logps/rejected": -101.36627197265625, | |
| "loss": 0.1484, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.709464192390442, | |
| "rewards/margins": 2.303464889526367, | |
| "rewards/rejected": -0.5940006375312805, | |
| "step": 552 | |
| }, | |
| { | |
| "epoch": 35.3, | |
| "grad_norm": 2.487671136856079, | |
| "learning_rate": 4.558446786419045e-07, | |
| "logits/chosen": -2.6454193592071533, | |
| "logits/rejected": -2.667191982269287, | |
| "logps/chosen": -112.91888427734375, | |
| "logps/rejected": -100.34318542480469, | |
| "loss": 0.1571, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.6041696071624756, | |
| "rewards/margins": 2.3108744621276855, | |
| "rewards/rejected": -0.7067050337791443, | |
| "step": 556 | |
| }, | |
| { | |
| "epoch": 35.56, | |
| "grad_norm": 2.396897315979004, | |
| "learning_rate": 4.552012722189099e-07, | |
| "logits/chosen": -2.6444239616394043, | |
| "logits/rejected": -2.6464180946350098, | |
| "logps/chosen": -115.7559814453125, | |
| "logps/rejected": -111.63291931152344, | |
| "loss": 0.1752, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7378344535827637, | |
| "rewards/margins": 2.1783785820007324, | |
| "rewards/rejected": -0.4405441880226135, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 35.56, | |
| "eval_logits/chosen": -2.746135711669922, | |
| "eval_logits/rejected": -2.8161184787750244, | |
| "eval_logps/chosen": -121.27652740478516, | |
| "eval_logps/rejected": -99.45245361328125, | |
| "eval_loss": 0.38749733567237854, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.2792812585830688, | |
| "eval_rewards/margins": 1.4136368036270142, | |
| "eval_rewards/rejected": -0.1343555599451065, | |
| "eval_runtime": 19.3831, | |
| "eval_samples_per_second": 1.445, | |
| "eval_steps_per_second": 1.445, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 35.81, | |
| "grad_norm": 3.2688581943511963, | |
| "learning_rate": 4.5455367294475655e-07, | |
| "logits/chosen": -2.7136824131011963, | |
| "logits/rejected": -2.724392890930176, | |
| "logps/chosen": -114.86489868164062, | |
| "logps/rejected": -108.21212768554688, | |
| "loss": 0.1485, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7876478433609009, | |
| "rewards/margins": 2.3711612224578857, | |
| "rewards/rejected": -0.5835133790969849, | |
| "step": 564 | |
| }, | |
| { | |
| "epoch": 36.06, | |
| "grad_norm": 2.39674711227417, | |
| "learning_rate": 4.5390189405175715e-07, | |
| "logits/chosen": -2.6487505435943604, | |
| "logits/rejected": -2.651470899581909, | |
| "logps/chosen": -111.85699462890625, | |
| "logps/rejected": -101.57424926757812, | |
| "loss": 0.1222, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7182648181915283, | |
| "rewards/margins": 2.550403356552124, | |
| "rewards/rejected": -0.8321384787559509, | |
| "step": 568 | |
| }, | |
| { | |
| "epoch": 36.32, | |
| "grad_norm": 2.970404624938965, | |
| "learning_rate": 4.5324594885762576e-07, | |
| "logits/chosen": -2.670048952102661, | |
| "logits/rejected": -2.70831036567688, | |
| "logps/chosen": -113.16908264160156, | |
| "logps/rejected": -116.0752944946289, | |
| "loss": 0.1838, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.6036030054092407, | |
| "rewards/margins": 2.133262872695923, | |
| "rewards/rejected": -0.5296599268913269, | |
| "step": 572 | |
| }, | |
| { | |
| "epoch": 36.57, | |
| "grad_norm": 3.4210455417633057, | |
| "learning_rate": 4.5258585076520595e-07, | |
| "logits/chosen": -2.647081136703491, | |
| "logits/rejected": -2.625157594680786, | |
| "logps/chosen": -119.30801391601562, | |
| "logps/rejected": -101.5343017578125, | |
| "loss": 0.1394, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7498310804367065, | |
| "rewards/margins": 2.5768487453460693, | |
| "rewards/rejected": -0.8270176649093628, | |
| "step": 576 | |
| }, | |
| { | |
| "epoch": 36.83, | |
| "grad_norm": 2.8785665035247803, | |
| "learning_rate": 4.5192161326219716e-07, | |
| "logits/chosen": -2.678386688232422, | |
| "logits/rejected": -2.6918272972106934, | |
| "logps/chosen": -111.96650695800781, | |
| "logps/rejected": -100.6183853149414, | |
| "loss": 0.1539, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.703883409500122, | |
| "rewards/margins": 2.2555694580078125, | |
| "rewards/rejected": -0.5516858696937561, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 37.08, | |
| "grad_norm": 1.953771710395813, | |
| "learning_rate": 4.512532499208786e-07, | |
| "logits/chosen": -2.622621774673462, | |
| "logits/rejected": -2.6352713108062744, | |
| "logps/chosen": -117.12508392333984, | |
| "logps/rejected": -106.53787231445312, | |
| "loss": 0.1165, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.857337474822998, | |
| "rewards/margins": 2.5984973907470703, | |
| "rewards/rejected": -0.7411599159240723, | |
| "step": 584 | |
| }, | |
| { | |
| "epoch": 37.33, | |
| "grad_norm": 1.7565056085586548, | |
| "learning_rate": 4.505807743978324e-07, | |
| "logits/chosen": -2.6742653846740723, | |
| "logits/rejected": -2.6801705360412598, | |
| "logps/chosen": -112.03057098388672, | |
| "logps/rejected": -106.16900634765625, | |
| "loss": 0.145, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8526411056518555, | |
| "rewards/margins": 2.4808454513549805, | |
| "rewards/rejected": -0.6282044649124146, | |
| "step": 588 | |
| }, | |
| { | |
| "epoch": 37.59, | |
| "grad_norm": 2.013282299041748, | |
| "learning_rate": 4.4990420043366415e-07, | |
| "logits/chosen": -2.6183061599731445, | |
| "logits/rejected": -2.6616299152374268, | |
| "logps/chosen": -112.98906707763672, | |
| "logps/rejected": -105.28535461425781, | |
| "loss": 0.1353, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7617923021316528, | |
| "rewards/margins": 2.430490016937256, | |
| "rewards/rejected": -0.6686975359916687, | |
| "step": 592 | |
| }, | |
| { | |
| "epoch": 37.84, | |
| "grad_norm": 2.2207536697387695, | |
| "learning_rate": 4.492235418527227e-07, | |
| "logits/chosen": -2.6781811714172363, | |
| "logits/rejected": -2.6666665077209473, | |
| "logps/chosen": -116.05792999267578, | |
| "logps/rejected": -107.16488647460938, | |
| "loss": 0.1468, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7365107536315918, | |
| "rewards/margins": 2.3593077659606934, | |
| "rewards/rejected": -0.6227970719337463, | |
| "step": 596 | |
| }, | |
| { | |
| "epoch": 38.1, | |
| "grad_norm": 2.3460826873779297, | |
| "learning_rate": 4.48538812562817e-07, | |
| "logits/chosen": -2.63061785697937, | |
| "logits/rejected": -2.6308352947235107, | |
| "logps/chosen": -111.25851440429688, | |
| "logps/rejected": -103.23674011230469, | |
| "loss": 0.1414, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.6231989860534668, | |
| "rewards/margins": 2.478404998779297, | |
| "rewards/rejected": -0.8552060127258301, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 38.1, | |
| "eval_logits/chosen": -2.7357699871063232, | |
| "eval_logits/rejected": -2.804048538208008, | |
| "eval_logps/chosen": -121.1922607421875, | |
| "eval_logps/rejected": -100.1863021850586, | |
| "eval_loss": 0.38548800349235535, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.2877070903778076, | |
| "eval_rewards/margins": 1.495449423789978, | |
| "eval_rewards/rejected": -0.207742378115654, | |
| "eval_runtime": 19.3741, | |
| "eval_samples_per_second": 1.445, | |
| "eval_steps_per_second": 1.445, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 38.35, | |
| "grad_norm": 2.2148759365081787, | |
| "learning_rate": 4.4785002655493244e-07, | |
| "logits/chosen": -2.68538236618042, | |
| "logits/rejected": -2.721736431121826, | |
| "logps/chosen": -115.57979583740234, | |
| "logps/rejected": -107.06050872802734, | |
| "loss": 0.1434, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.6038768291473389, | |
| "rewards/margins": 2.4257020950317383, | |
| "rewards/rejected": -0.8218252658843994, | |
| "step": 604 | |
| }, | |
| { | |
| "epoch": 38.6, | |
| "grad_norm": 2.045954465866089, | |
| "learning_rate": 4.4715719790294473e-07, | |
| "logits/chosen": -2.665043592453003, | |
| "logits/rejected": -2.6772406101226807, | |
| "logps/chosen": -118.3508071899414, | |
| "logps/rejected": -105.97952270507812, | |
| "loss": 0.1027, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.952754259109497, | |
| "rewards/margins": 2.809091329574585, | |
| "rewards/rejected": -0.8563370108604431, | |
| "step": 608 | |
| }, | |
| { | |
| "epoch": 38.86, | |
| "grad_norm": 3.2380034923553467, | |
| "learning_rate": 4.4646034076333254e-07, | |
| "logits/chosen": -2.6413214206695557, | |
| "logits/rejected": -2.614654064178467, | |
| "logps/chosen": -112.02613067626953, | |
| "logps/rejected": -107.12358856201172, | |
| "loss": 0.1423, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.760948657989502, | |
| "rewards/margins": 2.4380459785461426, | |
| "rewards/rejected": -0.6770972609519958, | |
| "step": 612 | |
| }, | |
| { | |
| "epoch": 39.11, | |
| "grad_norm": 1.8163915872573853, | |
| "learning_rate": 4.4575946937488807e-07, | |
| "logits/chosen": -2.653783082962036, | |
| "logits/rejected": -2.697185516357422, | |
| "logps/chosen": -109.09898376464844, | |
| "logps/rejected": -105.73826599121094, | |
| "loss": 0.1531, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.7506372928619385, | |
| "rewards/margins": 2.376413345336914, | |
| "rewards/rejected": -0.6257758140563965, | |
| "step": 616 | |
| }, | |
| { | |
| "epoch": 39.37, | |
| "grad_norm": 2.3035383224487305, | |
| "learning_rate": 4.4505459805842593e-07, | |
| "logits/chosen": -2.6597530841827393, | |
| "logits/rejected": -2.691429615020752, | |
| "logps/chosen": -108.01673889160156, | |
| "logps/rejected": -106.93608093261719, | |
| "loss": 0.1441, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7096694707870483, | |
| "rewards/margins": 2.528447389602661, | |
| "rewards/rejected": -0.8187777996063232, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 39.62, | |
| "grad_norm": 2.202437162399292, | |
| "learning_rate": 4.44345741216491e-07, | |
| "logits/chosen": -2.6261043548583984, | |
| "logits/rejected": -2.6052300930023193, | |
| "logps/chosen": -114.36175537109375, | |
| "logps/rejected": -107.27129364013672, | |
| "loss": 0.1432, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.74629807472229, | |
| "rewards/margins": 2.387127637863159, | |
| "rewards/rejected": -0.6408295035362244, | |
| "step": 624 | |
| }, | |
| { | |
| "epoch": 39.87, | |
| "grad_norm": 2.515286684036255, | |
| "learning_rate": 4.43632913333064e-07, | |
| "logits/chosen": -2.6591076850891113, | |
| "logits/rejected": -2.661803722381592, | |
| "logps/chosen": -115.1274185180664, | |
| "logps/rejected": -105.72164154052734, | |
| "loss": 0.1232, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.884786605834961, | |
| "rewards/margins": 2.571641206741333, | |
| "rewards/rejected": -0.6868546009063721, | |
| "step": 628 | |
| }, | |
| { | |
| "epoch": 40.13, | |
| "grad_norm": 2.1287059783935547, | |
| "learning_rate": 4.429161289732649e-07, | |
| "logits/chosen": -2.6356821060180664, | |
| "logits/rejected": -2.644843101501465, | |
| "logps/chosen": -117.84187316894531, | |
| "logps/rejected": -108.8236083984375, | |
| "loss": 0.1079, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9427728652954102, | |
| "rewards/margins": 2.9027700424194336, | |
| "rewards/rejected": -0.9599968194961548, | |
| "step": 632 | |
| }, | |
| { | |
| "epoch": 40.38, | |
| "grad_norm": 2.836397886276245, | |
| "learning_rate": 4.4219540278305647e-07, | |
| "logits/chosen": -2.681767463684082, | |
| "logits/rejected": -2.682898998260498, | |
| "logps/chosen": -108.63336181640625, | |
| "logps/rejected": -104.24710845947266, | |
| "loss": 0.1326, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.57785165309906, | |
| "rewards/margins": 2.463282346725464, | |
| "rewards/rejected": -0.8854306936264038, | |
| "step": 636 | |
| }, | |
| { | |
| "epoch": 40.63, | |
| "grad_norm": 2.018007516860962, | |
| "learning_rate": 4.414707494889439e-07, | |
| "logits/chosen": -2.619904041290283, | |
| "logits/rejected": -2.6312315464019775, | |
| "logps/chosen": -107.43942260742188, | |
| "logps/rejected": -104.7081069946289, | |
| "loss": 0.1121, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8468282222747803, | |
| "rewards/margins": 2.7092161178588867, | |
| "rewards/rejected": -0.8623881340026855, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 40.63, | |
| "eval_logits/chosen": -2.7261669635772705, | |
| "eval_logits/rejected": -2.792802572250366, | |
| "eval_logps/chosen": -121.07279205322266, | |
| "eval_logps/rejected": -100.8832778930664, | |
| "eval_loss": 0.3840921223163605, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.2996548414230347, | |
| "eval_rewards/margins": 1.5770933628082275, | |
| "eval_rewards/rejected": -0.2774384915828705, | |
| "eval_runtime": 19.3804, | |
| "eval_samples_per_second": 1.445, | |
| "eval_steps_per_second": 1.445, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 40.89, | |
| "grad_norm": 2.168912410736084, | |
| "learning_rate": 4.4074218389767466e-07, | |
| "logits/chosen": -2.6166744232177734, | |
| "logits/rejected": -2.6106488704681396, | |
| "logps/chosen": -124.27295684814453, | |
| "logps/rejected": -111.69468688964844, | |
| "loss": 0.1185, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9064750671386719, | |
| "rewards/margins": 2.79134464263916, | |
| "rewards/rejected": -0.8848695755004883, | |
| "step": 644 | |
| }, | |
| { | |
| "epoch": 41.14, | |
| "grad_norm": 1.7980531454086304, | |
| "learning_rate": 4.400097208959356e-07, | |
| "logits/chosen": -2.6521995067596436, | |
| "logits/rejected": -2.704921245574951, | |
| "logps/chosen": -114.37103271484375, | |
| "logps/rejected": -113.26276397705078, | |
| "loss": 0.1368, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.8770157098770142, | |
| "rewards/margins": 2.7274460792541504, | |
| "rewards/rejected": -0.8504302501678467, | |
| "step": 648 | |
| }, | |
| { | |
| "epoch": 41.4, | |
| "grad_norm": 2.447611093521118, | |
| "learning_rate": 4.3927337545004894e-07, | |
| "logits/chosen": -2.6812400817871094, | |
| "logits/rejected": -2.725090503692627, | |
| "logps/chosen": -112.9039535522461, | |
| "logps/rejected": -103.30266571044922, | |
| "loss": 0.119, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7921515703201294, | |
| "rewards/margins": 2.6354293823242188, | |
| "rewards/rejected": -0.8432780504226685, | |
| "step": 652 | |
| }, | |
| { | |
| "epoch": 41.65, | |
| "grad_norm": 2.0098862648010254, | |
| "learning_rate": 4.385331626056663e-07, | |
| "logits/chosen": -2.6464550495147705, | |
| "logits/rejected": -2.60159969329834, | |
| "logps/chosen": -118.65347290039062, | |
| "logps/rejected": -106.13660430908203, | |
| "loss": 0.1482, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.773016333580017, | |
| "rewards/margins": 2.4340577125549316, | |
| "rewards/rejected": -0.661041259765625, | |
| "step": 656 | |
| }, | |
| { | |
| "epoch": 41.9, | |
| "grad_norm": 2.093533515930176, | |
| "learning_rate": 4.377890974874613e-07, | |
| "logits/chosen": -2.652855157852173, | |
| "logits/rejected": -2.6764514446258545, | |
| "logps/chosen": -110.1133804321289, | |
| "logps/rejected": -112.76753997802734, | |
| "loss": 0.1062, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.795628309249878, | |
| "rewards/margins": 2.800370693206787, | |
| "rewards/rejected": -1.0047426223754883, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 42.16, | |
| "grad_norm": 1.8786094188690186, | |
| "learning_rate": 4.370411952988206e-07, | |
| "logits/chosen": -2.5962324142456055, | |
| "logits/rejected": -2.580698251724243, | |
| "logps/chosen": -116.74523162841797, | |
| "logps/rejected": -108.6583251953125, | |
| "loss": 0.1241, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7898542881011963, | |
| "rewards/margins": 2.6399893760681152, | |
| "rewards/rejected": -0.8501349687576294, | |
| "step": 664 | |
| }, | |
| { | |
| "epoch": 42.41, | |
| "grad_norm": 1.1881071329116821, | |
| "learning_rate": 4.362894713215334e-07, | |
| "logits/chosen": -2.642057418823242, | |
| "logits/rejected": -2.6707074642181396, | |
| "logps/chosen": -114.59297180175781, | |
| "logps/rejected": -107.72032165527344, | |
| "loss": 0.1115, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9477055072784424, | |
| "rewards/margins": 2.8435428142547607, | |
| "rewards/rejected": -0.8958378434181213, | |
| "step": 668 | |
| }, | |
| { | |
| "epoch": 42.67, | |
| "grad_norm": 2.0914270877838135, | |
| "learning_rate": 4.355339409154788e-07, | |
| "logits/chosen": -2.650442123413086, | |
| "logits/rejected": -2.647216558456421, | |
| "logps/chosen": -108.08013916015625, | |
| "logps/rejected": -103.83348083496094, | |
| "loss": 0.093, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8842676877975464, | |
| "rewards/margins": 2.963129997253418, | |
| "rewards/rejected": -1.0788623094558716, | |
| "step": 672 | |
| }, | |
| { | |
| "epoch": 42.92, | |
| "grad_norm": 2.257300615310669, | |
| "learning_rate": 4.347746195183122e-07, | |
| "logits/chosen": -2.6150929927825928, | |
| "logits/rejected": -2.6115312576293945, | |
| "logps/chosen": -113.52247619628906, | |
| "logps/rejected": -110.75169372558594, | |
| "loss": 0.1205, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.8281488418579102, | |
| "rewards/margins": 2.7573764324188232, | |
| "rewards/rejected": -0.9292274117469788, | |
| "step": 676 | |
| }, | |
| { | |
| "epoch": 43.17, | |
| "grad_norm": 1.82343327999115, | |
| "learning_rate": 4.340115226451501e-07, | |
| "logits/chosen": -2.6201157569885254, | |
| "logits/rejected": -2.6617937088012695, | |
| "logps/chosen": -108.35755157470703, | |
| "logps/rejected": -107.59652709960938, | |
| "loss": 0.0976, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8024065494537354, | |
| "rewards/margins": 2.9102468490600586, | |
| "rewards/rejected": -1.1078401803970337, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 43.17, | |
| "eval_logits/chosen": -2.7156684398651123, | |
| "eval_logits/rejected": -2.7805325984954834, | |
| "eval_logps/chosen": -121.03025817871094, | |
| "eval_logps/rejected": -101.69651794433594, | |
| "eval_loss": 0.38208508491516113, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.3039093017578125, | |
| "eval_rewards/margins": 1.6626710891723633, | |
| "eval_rewards/rejected": -0.358761727809906, | |
| "eval_runtime": 19.3666, | |
| "eval_samples_per_second": 1.446, | |
| "eval_steps_per_second": 1.446, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 43.43, | |
| "grad_norm": 1.9972604513168335, | |
| "learning_rate": 4.3324466588825223e-07, | |
| "logits/chosen": -2.6469597816467285, | |
| "logits/rejected": -2.6479005813598633, | |
| "logps/chosen": -117.90531921386719, | |
| "logps/rejected": -106.33170318603516, | |
| "loss": 0.1079, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8468865156173706, | |
| "rewards/margins": 2.780641555786133, | |
| "rewards/rejected": -0.9337549805641174, | |
| "step": 684 | |
| }, | |
| { | |
| "epoch": 43.68, | |
| "grad_norm": 1.7684298753738403, | |
| "learning_rate": 4.3247406491670435e-07, | |
| "logits/chosen": -2.653805732727051, | |
| "logits/rejected": -2.658804178237915, | |
| "logps/chosen": -108.92557525634766, | |
| "logps/rejected": -110.38461303710938, | |
| "loss": 0.1272, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9131863117218018, | |
| "rewards/margins": 2.63901948928833, | |
| "rewards/rejected": -0.7258330583572388, | |
| "step": 688 | |
| }, | |
| { | |
| "epoch": 43.94, | |
| "grad_norm": 1.980388879776001, | |
| "learning_rate": 4.3169973547609644e-07, | |
| "logits/chosen": -2.637269973754883, | |
| "logits/rejected": -2.6658825874328613, | |
| "logps/chosen": -110.72245788574219, | |
| "logps/rejected": -109.98155975341797, | |
| "loss": 0.1142, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.804459810256958, | |
| "rewards/margins": 2.8206331729888916, | |
| "rewards/rejected": -1.0161734819412231, | |
| "step": 692 | |
| }, | |
| { | |
| "epoch": 44.19, | |
| "grad_norm": 1.6069421768188477, | |
| "learning_rate": 4.3092169338820245e-07, | |
| "logits/chosen": -2.6507728099823, | |
| "logits/rejected": -2.5921573638916016, | |
| "logps/chosen": -115.70768737792969, | |
| "logps/rejected": -117.3571548461914, | |
| "loss": 0.1211, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.852295994758606, | |
| "rewards/margins": 2.9615478515625, | |
| "rewards/rejected": -1.1092519760131836, | |
| "step": 696 | |
| }, | |
| { | |
| "epoch": 44.44, | |
| "grad_norm": 2.384201765060425, | |
| "learning_rate": 4.30139954550656e-07, | |
| "logits/chosen": -2.6414434909820557, | |
| "logits/rejected": -2.638594627380371, | |
| "logps/chosen": -112.53083038330078, | |
| "logps/rejected": -103.69336700439453, | |
| "loss": 0.0863, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9861321449279785, | |
| "rewards/margins": 3.037813663482666, | |
| "rewards/rejected": -1.0516817569732666, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 44.7, | |
| "grad_norm": 2.6236279010772705, | |
| "learning_rate": 4.293545349366261e-07, | |
| "logits/chosen": -2.627302646636963, | |
| "logits/rejected": -2.648130416870117, | |
| "logps/chosen": -117.52839660644531, | |
| "logps/rejected": -113.75141143798828, | |
| "loss": 0.1266, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7409615516662598, | |
| "rewards/margins": 2.609776258468628, | |
| "rewards/rejected": -0.8688147068023682, | |
| "step": 704 | |
| }, | |
| { | |
| "epoch": 44.95, | |
| "grad_norm": 1.3566991090774536, | |
| "learning_rate": 4.2856545059449054e-07, | |
| "logits/chosen": -2.609771251678467, | |
| "logits/rejected": -2.649744749069214, | |
| "logps/chosen": -110.12776947021484, | |
| "logps/rejected": -104.61019897460938, | |
| "loss": 0.0946, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9009833335876465, | |
| "rewards/margins": 3.0006396770477295, | |
| "rewards/rejected": -1.099656343460083, | |
| "step": 708 | |
| }, | |
| { | |
| "epoch": 45.21, | |
| "grad_norm": 1.4097861051559448, | |
| "learning_rate": 4.27772717647508e-07, | |
| "logits/chosen": -2.639524459838867, | |
| "logits/rejected": -2.6454386711120605, | |
| "logps/chosen": -110.92047119140625, | |
| "logps/rejected": -104.04742431640625, | |
| "loss": 0.1045, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7894549369812012, | |
| "rewards/margins": 2.7535037994384766, | |
| "rewards/rejected": -0.9640489816665649, | |
| "step": 712 | |
| }, | |
| { | |
| "epoch": 45.46, | |
| "grad_norm": 2.119607448577881, | |
| "learning_rate": 4.2697635229348874e-07, | |
| "logits/chosen": -2.6511929035186768, | |
| "logits/rejected": -2.633582353591919, | |
| "logps/chosen": -109.69573974609375, | |
| "logps/rejected": -110.53610229492188, | |
| "loss": 0.1163, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7527060508728027, | |
| "rewards/margins": 2.6145551204681396, | |
| "rewards/rejected": -0.8618494868278503, | |
| "step": 716 | |
| }, | |
| { | |
| "epoch": 45.71, | |
| "grad_norm": 1.88676917552948, | |
| "learning_rate": 4.261763708044632e-07, | |
| "logits/chosen": -2.637509822845459, | |
| "logits/rejected": -2.6555514335632324, | |
| "logps/chosen": -114.49153137207031, | |
| "logps/rejected": -110.72467041015625, | |
| "loss": 0.1129, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.9959712028503418, | |
| "rewards/margins": 3.11696720123291, | |
| "rewards/rejected": -1.1209962368011475, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 45.71, | |
| "eval_logits/chosen": -2.704455614089966, | |
| "eval_logits/rejected": -2.7675421237945557, | |
| "eval_logps/chosen": -121.06656646728516, | |
| "eval_logps/rejected": -102.52320098876953, | |
| "eval_loss": 0.38231486082077026, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.3002774715423584, | |
| "eval_rewards/margins": 1.741708517074585, | |
| "eval_rewards/rejected": -0.44143104553222656, | |
| "eval_runtime": 19.3804, | |
| "eval_samples_per_second": 1.445, | |
| "eval_steps_per_second": 1.445, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 45.97, | |
| "grad_norm": 1.0807371139526367, | |
| "learning_rate": 4.2537278952635036e-07, | |
| "logits/chosen": -2.653682231903076, | |
| "logits/rejected": -2.6631548404693604, | |
| "logps/chosen": -112.33302307128906, | |
| "logps/rejected": -112.11505889892578, | |
| "loss": 0.0763, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9477115869522095, | |
| "rewards/margins": 3.1752381324768066, | |
| "rewards/rejected": -1.227526307106018, | |
| "step": 724 | |
| }, | |
| { | |
| "epoch": 46.22, | |
| "grad_norm": 1.5240466594696045, | |
| "learning_rate": 4.245656248786228e-07, | |
| "logits/chosen": -2.5998778343200684, | |
| "logits/rejected": -2.628650188446045, | |
| "logps/chosen": -116.5973892211914, | |
| "logps/rejected": -119.11909484863281, | |
| "loss": 0.0753, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.059324026107788, | |
| "rewards/margins": 3.2868502140045166, | |
| "rewards/rejected": -1.2275261878967285, | |
| "step": 728 | |
| }, | |
| { | |
| "epoch": 46.48, | |
| "grad_norm": 1.6910673379898071, | |
| "learning_rate": 4.237548933539718e-07, | |
| "logits/chosen": -2.609449863433838, | |
| "logits/rejected": -2.6589772701263428, | |
| "logps/chosen": -110.6764907836914, | |
| "logps/rejected": -105.00558471679688, | |
| "loss": 0.1179, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.9273223876953125, | |
| "rewards/margins": 2.8337433338165283, | |
| "rewards/rejected": -0.9064207077026367, | |
| "step": 732 | |
| }, | |
| { | |
| "epoch": 46.73, | |
| "grad_norm": 1.9393585920333862, | |
| "learning_rate": 4.229406115179702e-07, | |
| "logits/chosen": -2.674577474594116, | |
| "logits/rejected": -2.6538712978363037, | |
| "logps/chosen": -114.55770874023438, | |
| "logps/rejected": -111.56117248535156, | |
| "loss": 0.1062, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.6356512308120728, | |
| "rewards/margins": 2.7821154594421387, | |
| "rewards/rejected": -1.146464467048645, | |
| "step": 736 | |
| }, | |
| { | |
| "epoch": 46.98, | |
| "grad_norm": 1.7621327638626099, | |
| "learning_rate": 4.221227960087338e-07, | |
| "logits/chosen": -2.6068484783172607, | |
| "logits/rejected": -2.5801186561584473, | |
| "logps/chosen": -109.57792663574219, | |
| "logps/rejected": -106.45953369140625, | |
| "loss": 0.0958, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8922866582870483, | |
| "rewards/margins": 3.114880323410034, | |
| "rewards/rejected": -1.2225935459136963, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 47.24, | |
| "grad_norm": 1.7458181381225586, | |
| "learning_rate": 4.213014635365815e-07, | |
| "logits/chosen": -2.6573941707611084, | |
| "logits/rejected": -2.653240919113159, | |
| "logps/chosen": -116.97773742675781, | |
| "logps/rejected": -108.12200164794922, | |
| "loss": 0.0816, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0080180168151855, | |
| "rewards/margins": 3.296840190887451, | |
| "rewards/rejected": -1.2888221740722656, | |
| "step": 744 | |
| }, | |
| { | |
| "epoch": 47.49, | |
| "grad_norm": 1.9578934907913208, | |
| "learning_rate": 4.20476630883694e-07, | |
| "logits/chosen": -2.5878279209136963, | |
| "logits/rejected": -2.595855712890625, | |
| "logps/chosen": -115.75021362304688, | |
| "logps/rejected": -112.4002685546875, | |
| "loss": 0.075, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9529811143875122, | |
| "rewards/margins": 3.2216196060180664, | |
| "rewards/rejected": -1.2686389684677124, | |
| "step": 748 | |
| }, | |
| { | |
| "epoch": 47.75, | |
| "grad_norm": 1.6000980138778687, | |
| "learning_rate": 4.196483149037706e-07, | |
| "logits/chosen": -2.632962226867676, | |
| "logits/rejected": -2.6356112957000732, | |
| "logps/chosen": -110.47245788574219, | |
| "logps/rejected": -111.49674224853516, | |
| "loss": 0.1139, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.7682106494903564, | |
| "rewards/margins": 2.8430519104003906, | |
| "rewards/rejected": -1.074841022491455, | |
| "step": 752 | |
| }, | |
| { | |
| "epoch": 48.0, | |
| "grad_norm": 1.9791028499603271, | |
| "learning_rate": 4.188165325216849e-07, | |
| "logits/chosen": -2.6407368183135986, | |
| "logits/rejected": -2.6439175605773926, | |
| "logps/chosen": -107.70930480957031, | |
| "logps/rejected": -111.42974853515625, | |
| "loss": 0.1044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9136321544647217, | |
| "rewards/margins": 3.008749485015869, | |
| "rewards/rejected": -1.095117211341858, | |
| "step": 756 | |
| }, | |
| { | |
| "epoch": 48.25, | |
| "grad_norm": 1.486619234085083, | |
| "learning_rate": 4.179813007331393e-07, | |
| "logits/chosen": -2.638111114501953, | |
| "logits/rejected": -2.6660499572753906, | |
| "logps/chosen": -112.31642150878906, | |
| "logps/rejected": -114.76564025878906, | |
| "loss": 0.1037, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7412384748458862, | |
| "rewards/margins": 2.972817897796631, | |
| "rewards/rejected": -1.2315791845321655, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 48.25, | |
| "eval_logits/chosen": -2.69281268119812, | |
| "eval_logits/rejected": -2.754284143447876, | |
| "eval_logps/chosen": -121.21448516845703, | |
| "eval_logps/rejected": -103.48255920410156, | |
| "eval_loss": 0.3811449110507965, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.2854851484298706, | |
| "eval_rewards/margins": 1.8228529691696167, | |
| "eval_rewards/rejected": -0.5373677015304565, | |
| "eval_runtime": 19.4147, | |
| "eval_samples_per_second": 1.442, | |
| "eval_steps_per_second": 1.442, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 48.51, | |
| "grad_norm": 1.6442840099334717, | |
| "learning_rate": 4.171426366043172e-07, | |
| "logits/chosen": -2.5812575817108154, | |
| "logits/rejected": -2.6027064323425293, | |
| "logps/chosen": -109.93325805664062, | |
| "logps/rejected": -106.68315887451172, | |
| "loss": 0.1014, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 2.025794506072998, | |
| "rewards/margins": 2.979997158050537, | |
| "rewards/rejected": -0.9542028903961182, | |
| "step": 764 | |
| }, | |
| { | |
| "epoch": 48.76, | |
| "grad_norm": 1.7033922672271729, | |
| "learning_rate": 4.163005572715348e-07, | |
| "logits/chosen": -2.6315484046936035, | |
| "logits/rejected": -2.652425765991211, | |
| "logps/chosen": -114.54792785644531, | |
| "logps/rejected": -113.0596923828125, | |
| "loss": 0.0923, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9352294206619263, | |
| "rewards/margins": 3.1911542415618896, | |
| "rewards/rejected": -1.2559247016906738, | |
| "step": 768 | |
| }, | |
| { | |
| "epoch": 49.02, | |
| "grad_norm": 1.4940606355667114, | |
| "learning_rate": 4.154550799408906e-07, | |
| "logits/chosen": -2.6459126472473145, | |
| "logits/rejected": -2.59468936920166, | |
| "logps/chosen": -112.78709411621094, | |
| "logps/rejected": -111.69085693359375, | |
| "loss": 0.0632, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9187610149383545, | |
| "rewards/margins": 3.4088921546936035, | |
| "rewards/rejected": -1.4901307821273804, | |
| "step": 772 | |
| }, | |
| { | |
| "epoch": 49.27, | |
| "grad_norm": 1.6349976062774658, | |
| "learning_rate": 4.1460622188791386e-07, | |
| "logits/chosen": -2.626100778579712, | |
| "logits/rejected": -2.6048431396484375, | |
| "logps/chosen": -114.90809631347656, | |
| "logps/rejected": -110.02332305908203, | |
| "loss": 0.0862, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.9543546438217163, | |
| "rewards/margins": 3.2480387687683105, | |
| "rewards/rejected": -1.2936842441558838, | |
| "step": 776 | |
| }, | |
| { | |
| "epoch": 49.52, | |
| "grad_norm": 1.4356471300125122, | |
| "learning_rate": 4.13754000457212e-07, | |
| "logits/chosen": -2.6456077098846436, | |
| "logits/rejected": -2.676055908203125, | |
| "logps/chosen": -103.83885955810547, | |
| "logps/rejected": -109.37602233886719, | |
| "loss": 0.082, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8857074975967407, | |
| "rewards/margins": 3.213439702987671, | |
| "rewards/rejected": -1.3277320861816406, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 49.78, | |
| "grad_norm": 1.7946281433105469, | |
| "learning_rate": 4.128984330621156e-07, | |
| "logits/chosen": -2.6110281944274902, | |
| "logits/rejected": -2.605884075164795, | |
| "logps/chosen": -116.66584777832031, | |
| "logps/rejected": -114.77125549316406, | |
| "loss": 0.0825, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0007410049438477, | |
| "rewards/margins": 3.1677379608154297, | |
| "rewards/rejected": -1.1669968366622925, | |
| "step": 784 | |
| }, | |
| { | |
| "epoch": 50.03, | |
| "grad_norm": 2.0904555320739746, | |
| "learning_rate": 4.1203953718432304e-07, | |
| "logits/chosen": -2.5824971199035645, | |
| "logits/rejected": -2.5949392318725586, | |
| "logps/chosen": -113.75666809082031, | |
| "logps/rejected": -112.34939575195312, | |
| "loss": 0.0968, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8396023511886597, | |
| "rewards/margins": 3.167187213897705, | |
| "rewards/rejected": -1.3275845050811768, | |
| "step": 788 | |
| }, | |
| { | |
| "epoch": 50.29, | |
| "grad_norm": 1.8408178091049194, | |
| "learning_rate": 4.1117733037354313e-07, | |
| "logits/chosen": -2.5902395248413086, | |
| "logits/rejected": -2.590879440307617, | |
| "logps/chosen": -116.93998718261719, | |
| "logps/rejected": -112.17693328857422, | |
| "loss": 0.0719, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.067030429840088, | |
| "rewards/margins": 3.2486207485198975, | |
| "rewards/rejected": -1.1815906763076782, | |
| "step": 792 | |
| }, | |
| { | |
| "epoch": 50.54, | |
| "grad_norm": 1.2143826484680176, | |
| "learning_rate": 4.1031183024713657e-07, | |
| "logits/chosen": -2.6065564155578613, | |
| "logits/rejected": -2.5807137489318848, | |
| "logps/chosen": -116.70026397705078, | |
| "logps/rejected": -109.03843688964844, | |
| "loss": 0.0901, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 2.0011649131774902, | |
| "rewards/margins": 3.269892454147339, | |
| "rewards/rejected": -1.2687275409698486, | |
| "step": 796 | |
| }, | |
| { | |
| "epoch": 50.79, | |
| "grad_norm": 1.2470941543579102, | |
| "learning_rate": 4.0944305448975594e-07, | |
| "logits/chosen": -2.6263880729675293, | |
| "logits/rejected": -2.6351208686828613, | |
| "logps/chosen": -109.2755126953125, | |
| "logps/rejected": -116.7694091796875, | |
| "loss": 0.085, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.79188871383667, | |
| "rewards/margins": 3.278684616088867, | |
| "rewards/rejected": -1.4867955446243286, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 50.79, | |
| "eval_logits/chosen": -2.681551694869995, | |
| "eval_logits/rejected": -2.741281270980835, | |
| "eval_logps/chosen": -121.30486297607422, | |
| "eval_logps/rejected": -104.38188171386719, | |
| "eval_loss": 0.38058552145957947, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.2764482498168945, | |
| "eval_rewards/margins": 1.9037466049194336, | |
| "eval_rewards/rejected": -0.6272983551025391, | |
| "eval_runtime": 19.3679, | |
| "eval_samples_per_second": 1.446, | |
| "eval_steps_per_second": 1.446, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 51.05, | |
| "grad_norm": 2.3190038204193115, | |
| "learning_rate": 4.085710208529843e-07, | |
| "logits/chosen": -2.6023600101470947, | |
| "logits/rejected": -2.6153862476348877, | |
| "logps/chosen": -105.47126007080078, | |
| "logps/rejected": -109.3538818359375, | |
| "loss": 0.0898, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8685368299484253, | |
| "rewards/margins": 3.1377530097961426, | |
| "rewards/rejected": -1.2692162990570068, | |
| "step": 804 | |
| }, | |
| { | |
| "epoch": 51.3, | |
| "grad_norm": 1.2653313875198364, | |
| "learning_rate": 4.076957471549728e-07, | |
| "logits/chosen": -2.6036365032196045, | |
| "logits/rejected": -2.623936414718628, | |
| "logps/chosen": -112.60127258300781, | |
| "logps/rejected": -108.7287368774414, | |
| "loss": 0.0865, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9202613830566406, | |
| "rewards/margins": 3.164761543273926, | |
| "rewards/rejected": -1.2445003986358643, | |
| "step": 808 | |
| }, | |
| { | |
| "epoch": 51.56, | |
| "grad_norm": 1.1972923278808594, | |
| "learning_rate": 4.068172512800759e-07, | |
| "logits/chosen": -2.5859198570251465, | |
| "logits/rejected": -2.5816516876220703, | |
| "logps/chosen": -110.08045959472656, | |
| "logps/rejected": -117.87619018554688, | |
| "loss": 0.0605, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0786402225494385, | |
| "rewards/margins": 3.6814520359039307, | |
| "rewards/rejected": -1.6028118133544922, | |
| "step": 812 | |
| }, | |
| { | |
| "epoch": 51.81, | |
| "grad_norm": 1.5057625770568848, | |
| "learning_rate": 4.059355511784868e-07, | |
| "logits/chosen": -2.619558572769165, | |
| "logits/rejected": -2.6305971145629883, | |
| "logps/chosen": -114.13056945800781, | |
| "logps/rejected": -113.92069244384766, | |
| "loss": 0.0933, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.7876722812652588, | |
| "rewards/margins": 3.144243001937866, | |
| "rewards/rejected": -1.3565707206726074, | |
| "step": 816 | |
| }, | |
| { | |
| "epoch": 52.06, | |
| "grad_norm": 1.5957279205322266, | |
| "learning_rate": 4.0505066486587e-07, | |
| "logits/chosen": -2.638758659362793, | |
| "logits/rejected": -2.615614175796509, | |
| "logps/chosen": -117.18038940429688, | |
| "logps/rejected": -118.73646545410156, | |
| "loss": 0.0666, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.000675678253174, | |
| "rewards/margins": 3.431554079055786, | |
| "rewards/rejected": -1.4308786392211914, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 52.32, | |
| "grad_norm": 1.7117719650268555, | |
| "learning_rate": 4.041626104229936e-07, | |
| "logits/chosen": -2.561399221420288, | |
| "logits/rejected": -2.5965375900268555, | |
| "logps/chosen": -108.37356567382812, | |
| "logps/rejected": -109.40271759033203, | |
| "loss": 0.0615, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.173064708709717, | |
| "rewards/margins": 3.6536784172058105, | |
| "rewards/rejected": -1.4806135892868042, | |
| "step": 824 | |
| }, | |
| { | |
| "epoch": 52.57, | |
| "grad_norm": 1.3572646379470825, | |
| "learning_rate": 4.032714059953595e-07, | |
| "logits/chosen": -2.6196985244750977, | |
| "logits/rejected": -2.614413022994995, | |
| "logps/chosen": -109.2529067993164, | |
| "logps/rejected": -104.56177520751953, | |
| "loss": 0.092, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.9012202024459839, | |
| "rewards/margins": 3.18809175491333, | |
| "rewards/rejected": -1.2868719100952148, | |
| "step": 828 | |
| }, | |
| { | |
| "epoch": 52.83, | |
| "grad_norm": 1.784955382347107, | |
| "learning_rate": 4.0237706979283305e-07, | |
| "logits/chosen": -2.6027987003326416, | |
| "logits/rejected": -2.6091623306274414, | |
| "logps/chosen": -116.41980743408203, | |
| "logps/rejected": -121.5534896850586, | |
| "loss": 0.0837, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.864450216293335, | |
| "rewards/margins": 3.336150646209717, | |
| "rewards/rejected": -1.4717004299163818, | |
| "step": 832 | |
| }, | |
| { | |
| "epoch": 53.08, | |
| "grad_norm": 1.603405237197876, | |
| "learning_rate": 4.014796200892706e-07, | |
| "logits/chosen": -2.59523344039917, | |
| "logits/rejected": -2.59432053565979, | |
| "logps/chosen": -108.4609603881836, | |
| "logps/rejected": -112.68742370605469, | |
| "loss": 0.071, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.829506278038025, | |
| "rewards/margins": 3.3810737133026123, | |
| "rewards/rejected": -1.5515676736831665, | |
| "step": 836 | |
| }, | |
| { | |
| "epoch": 53.33, | |
| "grad_norm": 1.8830804824829102, | |
| "learning_rate": 4.005790752221464e-07, | |
| "logits/chosen": -2.6272573471069336, | |
| "logits/rejected": -2.627049446105957, | |
| "logps/chosen": -113.10144805908203, | |
| "logps/rejected": -108.62581634521484, | |
| "loss": 0.0714, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.913501262664795, | |
| "rewards/margins": 3.577221632003784, | |
| "rewards/rejected": -1.663720726966858, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 53.33, | |
| "eval_logits/chosen": -2.668611526489258, | |
| "eval_logits/rejected": -2.726428508758545, | |
| "eval_logps/chosen": -121.56246185302734, | |
| "eval_logps/rejected": -105.41184997558594, | |
| "eval_loss": 0.381181925535202, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.2506877183914185, | |
| "eval_rewards/margins": 1.9809836149215698, | |
| "eval_rewards/rejected": -0.7302957773208618, | |
| "eval_runtime": 19.382, | |
| "eval_samples_per_second": 1.445, | |
| "eval_steps_per_second": 1.445, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 53.59, | |
| "grad_norm": 1.5219979286193848, | |
| "learning_rate": 3.996754535921777e-07, | |
| "logits/chosen": -2.594785451889038, | |
| "logits/rejected": -2.600487470626831, | |
| "logps/chosen": -106.41715240478516, | |
| "logps/rejected": -120.44597625732422, | |
| "loss": 0.0765, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9574780464172363, | |
| "rewards/margins": 3.235558032989502, | |
| "rewards/rejected": -1.2780797481536865, | |
| "step": 844 | |
| }, | |
| { | |
| "epoch": 53.84, | |
| "grad_norm": 0.8998934626579285, | |
| "learning_rate": 3.987687736629487e-07, | |
| "logits/chosen": -2.5739502906799316, | |
| "logits/rejected": -2.540557861328125, | |
| "logps/chosen": -116.07780456542969, | |
| "logps/rejected": -113.49636840820312, | |
| "loss": 0.0588, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.980076551437378, | |
| "rewards/margins": 3.6954421997070312, | |
| "rewards/rejected": -1.7153657674789429, | |
| "step": 848 | |
| }, | |
| { | |
| "epoch": 54.1, | |
| "grad_norm": 1.3017789125442505, | |
| "learning_rate": 3.9785905396053377e-07, | |
| "logits/chosen": -2.5990142822265625, | |
| "logits/rejected": -2.6168363094329834, | |
| "logps/chosen": -111.93960571289062, | |
| "logps/rejected": -116.76874542236328, | |
| "loss": 0.0872, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 2.0168068408966064, | |
| "rewards/margins": 3.3573083877563477, | |
| "rewards/rejected": -1.340501308441162, | |
| "step": 852 | |
| }, | |
| { | |
| "epoch": 54.35, | |
| "grad_norm": 1.2396210432052612, | |
| "learning_rate": 3.9694631307311825e-07, | |
| "logits/chosen": -2.5667662620544434, | |
| "logits/rejected": -2.5850954055786133, | |
| "logps/chosen": -111.22042846679688, | |
| "logps/rejected": -111.89250183105469, | |
| "loss": 0.0731, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9918500185012817, | |
| "rewards/margins": 3.535062313079834, | |
| "rewards/rejected": -1.5432121753692627, | |
| "step": 856 | |
| }, | |
| { | |
| "epoch": 54.6, | |
| "grad_norm": 1.335559606552124, | |
| "learning_rate": 3.960305696506192e-07, | |
| "logits/chosen": -2.5854077339172363, | |
| "logits/rejected": -2.5688228607177734, | |
| "logps/chosen": -116.70706939697266, | |
| "logps/rejected": -110.84555053710938, | |
| "loss": 0.0528, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.941176414489746, | |
| "rewards/margins": 3.603893518447876, | |
| "rewards/rejected": -1.6627171039581299, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 54.86, | |
| "grad_norm": 1.6855989694595337, | |
| "learning_rate": 3.95111842404304e-07, | |
| "logits/chosen": -2.5832574367523193, | |
| "logits/rejected": -2.5922787189483643, | |
| "logps/chosen": -115.02678680419922, | |
| "logps/rejected": -123.42115783691406, | |
| "loss": 0.0682, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9162797927856445, | |
| "rewards/margins": 3.6613926887512207, | |
| "rewards/rejected": -1.7451128959655762, | |
| "step": 864 | |
| }, | |
| { | |
| "epoch": 55.11, | |
| "grad_norm": 1.599083423614502, | |
| "learning_rate": 3.94190150106408e-07, | |
| "logits/chosen": -2.6082019805908203, | |
| "logits/rejected": -2.583937168121338, | |
| "logps/chosen": -106.68289947509766, | |
| "logps/rejected": -116.42233276367188, | |
| "loss": 0.0741, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8997607231140137, | |
| "rewards/margins": 3.2817296981811523, | |
| "rewards/rejected": -1.3819692134857178, | |
| "step": 868 | |
| }, | |
| { | |
| "epoch": 55.37, | |
| "grad_norm": 1.4026548862457275, | |
| "learning_rate": 3.9326551158975124e-07, | |
| "logits/chosen": -2.575761079788208, | |
| "logits/rejected": -2.6161446571350098, | |
| "logps/chosen": -109.90505981445312, | |
| "logps/rejected": -113.70225524902344, | |
| "loss": 0.0556, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.069807291030884, | |
| "rewards/margins": 3.6844887733459473, | |
| "rewards/rejected": -1.6146814823150635, | |
| "step": 872 | |
| }, | |
| { | |
| "epoch": 55.62, | |
| "grad_norm": 1.6232682466506958, | |
| "learning_rate": 3.923379457473534e-07, | |
| "logits/chosen": -2.569347858428955, | |
| "logits/rejected": -2.5830793380737305, | |
| "logps/chosen": -115.759033203125, | |
| "logps/rejected": -114.3780288696289, | |
| "loss": 0.0651, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.960666298866272, | |
| "rewards/margins": 3.5167884826660156, | |
| "rewards/rejected": -1.5561223030090332, | |
| "step": 876 | |
| }, | |
| { | |
| "epoch": 55.87, | |
| "grad_norm": 1.7684590816497803, | |
| "learning_rate": 3.914074715320479e-07, | |
| "logits/chosen": -2.5655999183654785, | |
| "logits/rejected": -2.5638012886047363, | |
| "logps/chosen": -109.46947479248047, | |
| "logps/rejected": -118.88379669189453, | |
| "loss": 0.0605, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8757377862930298, | |
| "rewards/margins": 3.77754807472229, | |
| "rewards/rejected": -1.9018104076385498, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 55.87, | |
| "eval_logits/chosen": -2.656400203704834, | |
| "eval_logits/rejected": -2.712291955947876, | |
| "eval_logps/chosen": -121.80836486816406, | |
| "eval_logps/rejected": -106.49579620361328, | |
| "eval_loss": 0.380399227142334, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.2260981798171997, | |
| "eval_rewards/margins": 2.0647895336151123, | |
| "eval_rewards/rejected": -0.8386915922164917, | |
| "eval_runtime": 19.382, | |
| "eval_samples_per_second": 1.445, | |
| "eval_steps_per_second": 1.445, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 56.13, | |
| "grad_norm": 1.283678412437439, | |
| "learning_rate": 3.9047410795609437e-07, | |
| "logits/chosen": -2.621877908706665, | |
| "logits/rejected": -2.5620386600494385, | |
| "logps/chosen": -112.5902328491211, | |
| "logps/rejected": -115.07412719726562, | |
| "loss": 0.0779, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.8606853485107422, | |
| "rewards/margins": 3.527510166168213, | |
| "rewards/rejected": -1.6668250560760498, | |
| "step": 884 | |
| }, | |
| { | |
| "epoch": 56.38, | |
| "grad_norm": 1.6468217372894287, | |
| "learning_rate": 3.8953787409079076e-07, | |
| "logits/chosen": -2.571714401245117, | |
| "logits/rejected": -2.581312417984009, | |
| "logps/chosen": -111.61944580078125, | |
| "logps/rejected": -115.42122650146484, | |
| "loss": 0.0643, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9630603790283203, | |
| "rewards/margins": 3.626448154449463, | |
| "rewards/rejected": -1.6633877754211426, | |
| "step": 888 | |
| }, | |
| { | |
| "epoch": 56.63, | |
| "grad_norm": 1.1661204099655151, | |
| "learning_rate": 3.885987890660827e-07, | |
| "logits/chosen": -2.5704822540283203, | |
| "logits/rejected": -2.5689477920532227, | |
| "logps/chosen": -109.46009063720703, | |
| "logps/rejected": -111.1002426147461, | |
| "loss": 0.0596, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0093395709991455, | |
| "rewards/margins": 3.5506997108459473, | |
| "rewards/rejected": -1.5413601398468018, | |
| "step": 892 | |
| }, | |
| { | |
| "epoch": 56.89, | |
| "grad_norm": 1.340222716331482, | |
| "learning_rate": 3.876568720701737e-07, | |
| "logits/chosen": -2.621769428253174, | |
| "logits/rejected": -2.6131060123443604, | |
| "logps/chosen": -114.04650115966797, | |
| "logps/rejected": -119.44813537597656, | |
| "loss": 0.0678, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.858720302581787, | |
| "rewards/margins": 3.575531244277954, | |
| "rewards/rejected": -1.716811180114746, | |
| "step": 896 | |
| }, | |
| { | |
| "epoch": 57.14, | |
| "grad_norm": 1.60097074508667, | |
| "learning_rate": 3.867121423491324e-07, | |
| "logits/chosen": -2.5544419288635254, | |
| "logits/rejected": -2.5702197551727295, | |
| "logps/chosen": -110.59100341796875, | |
| "logps/rejected": -116.37786102294922, | |
| "loss": 0.0639, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0666933059692383, | |
| "rewards/margins": 3.7600369453430176, | |
| "rewards/rejected": -1.6933435201644897, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 57.4, | |
| "grad_norm": 1.5483367443084717, | |
| "learning_rate": 3.857646192064995e-07, | |
| "logits/chosen": -2.5642123222351074, | |
| "logits/rejected": -2.5501718521118164, | |
| "logps/chosen": -110.6575698852539, | |
| "logps/rejected": -119.62319946289062, | |
| "loss": 0.0632, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.882343053817749, | |
| "rewards/margins": 3.3227930068969727, | |
| "rewards/rejected": -1.4404499530792236, | |
| "step": 904 | |
| }, | |
| { | |
| "epoch": 57.65, | |
| "grad_norm": 1.3593943119049072, | |
| "learning_rate": 3.848143220028931e-07, | |
| "logits/chosen": -2.6376965045928955, | |
| "logits/rejected": -2.618466377258301, | |
| "logps/chosen": -115.74939727783203, | |
| "logps/rejected": -112.37615966796875, | |
| "loss": 0.0644, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9629660844802856, | |
| "rewards/margins": 3.574894428253174, | |
| "rewards/rejected": -1.6119282245635986, | |
| "step": 908 | |
| }, | |
| { | |
| "epoch": 57.9, | |
| "grad_norm": 0.9320303201675415, | |
| "learning_rate": 3.8386127015561377e-07, | |
| "logits/chosen": -2.5762109756469727, | |
| "logits/rejected": -2.5863161087036133, | |
| "logps/chosen": -113.18948364257812, | |
| "logps/rejected": -116.29486846923828, | |
| "loss": 0.0671, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 2.0451762676239014, | |
| "rewards/margins": 3.999039649963379, | |
| "rewards/rejected": -1.9538631439208984, | |
| "step": 912 | |
| }, | |
| { | |
| "epoch": 58.16, | |
| "grad_norm": 1.352599859237671, | |
| "learning_rate": 3.82905483138247e-07, | |
| "logits/chosen": -2.550541400909424, | |
| "logits/rejected": -2.533012866973877, | |
| "logps/chosen": -114.77538299560547, | |
| "logps/rejected": -119.81204223632812, | |
| "loss": 0.0522, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9466917514801025, | |
| "rewards/margins": 4.070167541503906, | |
| "rewards/rejected": -2.123476028442383, | |
| "step": 916 | |
| }, | |
| { | |
| "epoch": 58.41, | |
| "grad_norm": 1.207864761352539, | |
| "learning_rate": 3.8194698048026583e-07, | |
| "logits/chosen": -2.5619962215423584, | |
| "logits/rejected": -2.569873094558716, | |
| "logps/chosen": -118.76224517822266, | |
| "logps/rejected": -118.89384460449219, | |
| "loss": 0.0526, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.794409155845642, | |
| "rewards/margins": 3.9158012866973877, | |
| "rewards/rejected": -2.121392250061035, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 58.41, | |
| "eval_logits/chosen": -2.6444404125213623, | |
| "eval_logits/rejected": -2.6984755992889404, | |
| "eval_logps/chosen": -122.1351318359375, | |
| "eval_logps/rejected": -107.59381103515625, | |
| "eval_loss": 0.38029375672340393, | |
| "eval_rewards/accuracies": 0.8214285969734192, | |
| "eval_rewards/chosen": 1.193420171737671, | |
| "eval_rewards/margins": 2.141911029815674, | |
| "eval_rewards/rejected": -0.9484910368919373, | |
| "eval_runtime": 19.4042, | |
| "eval_samples_per_second": 1.443, | |
| "eval_steps_per_second": 1.443, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 58.67, | |
| "grad_norm": 1.158010482788086, | |
| "learning_rate": 3.8098578176663154e-07, | |
| "logits/chosen": -2.585352659225464, | |
| "logits/rejected": -2.5922374725341797, | |
| "logps/chosen": -104.6133804321289, | |
| "logps/rejected": -110.61421203613281, | |
| "loss": 0.0737, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9587405920028687, | |
| "rewards/margins": 3.5576822757720947, | |
| "rewards/rejected": -1.598941683769226, | |
| "step": 924 | |
| }, | |
| { | |
| "epoch": 58.92, | |
| "grad_norm": 1.1459003686904907, | |
| "learning_rate": 3.800219066373936e-07, | |
| "logits/chosen": -2.636521816253662, | |
| "logits/rejected": -2.6258299350738525, | |
| "logps/chosen": -107.60220336914062, | |
| "logps/rejected": -120.26052856445312, | |
| "loss": 0.0492, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.1595146656036377, | |
| "rewards/margins": 3.6657819747924805, | |
| "rewards/rejected": -1.5062673091888428, | |
| "step": 928 | |
| }, | |
| { | |
| "epoch": 59.17, | |
| "grad_norm": 1.0757678747177124, | |
| "learning_rate": 3.7905537478728844e-07, | |
| "logits/chosen": -2.5207972526550293, | |
| "logits/rejected": -2.497737169265747, | |
| "logps/chosen": -112.18553161621094, | |
| "logps/rejected": -108.89622497558594, | |
| "loss": 0.0525, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0456583499908447, | |
| "rewards/margins": 3.8131721019744873, | |
| "rewards/rejected": -1.767513632774353, | |
| "step": 932 | |
| }, | |
| { | |
| "epoch": 59.43, | |
| "grad_norm": 1.0222171545028687, | |
| "learning_rate": 3.780862059653367e-07, | |
| "logits/chosen": -2.6137924194335938, | |
| "logits/rejected": -2.612893581390381, | |
| "logps/chosen": -110.01217651367188, | |
| "logps/rejected": -119.69099426269531, | |
| "loss": 0.0611, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.8505878448486328, | |
| "rewards/margins": 3.9199700355529785, | |
| "rewards/rejected": -2.0693817138671875, | |
| "step": 936 | |
| }, | |
| { | |
| "epoch": 59.68, | |
| "grad_norm": 1.1177177429199219, | |
| "learning_rate": 3.7711441997444016e-07, | |
| "logits/chosen": -2.538954257965088, | |
| "logits/rejected": -2.535083770751953, | |
| "logps/chosen": -117.10211944580078, | |
| "logps/rejected": -117.08683776855469, | |
| "loss": 0.0614, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.1291184425354004, | |
| "rewards/margins": 3.7359395027160645, | |
| "rewards/rejected": -1.606820821762085, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 59.94, | |
| "grad_norm": 1.25459885597229, | |
| "learning_rate": 3.7614003667097673e-07, | |
| "logits/chosen": -2.5723907947540283, | |
| "logits/rejected": -2.600084066390991, | |
| "logps/chosen": -106.81842041015625, | |
| "logps/rejected": -121.9610824584961, | |
| "loss": 0.0534, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7943745851516724, | |
| "rewards/margins": 3.866551399230957, | |
| "rewards/rejected": -2.0721771717071533, | |
| "step": 944 | |
| }, | |
| { | |
| "epoch": 60.19, | |
| "grad_norm": 1.1901063919067383, | |
| "learning_rate": 3.7516307596439494e-07, | |
| "logits/chosen": -2.5640013217926025, | |
| "logits/rejected": -2.5475199222564697, | |
| "logps/chosen": -116.98225402832031, | |
| "logps/rejected": -117.1365966796875, | |
| "loss": 0.0464, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.131293535232544, | |
| "rewards/margins": 4.100174903869629, | |
| "rewards/rejected": -1.9688811302185059, | |
| "step": 948 | |
| }, | |
| { | |
| "epoch": 60.44, | |
| "grad_norm": 1.3003920316696167, | |
| "learning_rate": 3.7418355781680707e-07, | |
| "logits/chosen": -2.5525896549224854, | |
| "logits/rejected": -2.5547983646392822, | |
| "logps/chosen": -111.23165893554688, | |
| "logps/rejected": -121.36713409423828, | |
| "loss": 0.065, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0756218433380127, | |
| "rewards/margins": 3.645637035369873, | |
| "rewards/rejected": -1.5700148344039917, | |
| "step": 952 | |
| }, | |
| { | |
| "epoch": 60.7, | |
| "grad_norm": 1.421809434890747, | |
| "learning_rate": 3.7320150224258116e-07, | |
| "logits/chosen": -2.5778801441192627, | |
| "logits/rejected": -2.585582733154297, | |
| "logps/chosen": -110.18714904785156, | |
| "logps/rejected": -124.43739318847656, | |
| "loss": 0.0465, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7850464582443237, | |
| "rewards/margins": 4.019922733306885, | |
| "rewards/rejected": -2.2348759174346924, | |
| "step": 956 | |
| }, | |
| { | |
| "epoch": 60.95, | |
| "grad_norm": 1.3294024467468262, | |
| "learning_rate": 3.7221692930793233e-07, | |
| "logits/chosen": -2.5990958213806152, | |
| "logits/rejected": -2.5878379344940186, | |
| "logps/chosen": -105.92156982421875, | |
| "logps/rejected": -110.94420623779297, | |
| "loss": 0.0601, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.8963432312011719, | |
| "rewards/margins": 3.7428231239318848, | |
| "rewards/rejected": -1.846480131149292, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 60.95, | |
| "eval_logits/chosen": -2.6317238807678223, | |
| "eval_logits/rejected": -2.6838760375976562, | |
| "eval_logps/chosen": -122.47028350830078, | |
| "eval_logps/rejected": -108.6797103881836, | |
| "eval_loss": 0.38127702474594116, | |
| "eval_rewards/accuracies": 0.8214285969734192, | |
| "eval_rewards/chosen": 1.1599055528640747, | |
| "eval_rewards/margins": 2.2169876098632812, | |
| "eval_rewards/rejected": -1.057082176208496, | |
| "eval_runtime": 19.3651, | |
| "eval_samples_per_second": 1.446, | |
| "eval_steps_per_second": 1.446, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 61.21, | |
| "grad_norm": 1.2264454364776611, | |
| "learning_rate": 3.7122985913051236e-07, | |
| "logits/chosen": -2.5557727813720703, | |
| "logits/rejected": -2.5267515182495117, | |
| "logps/chosen": -121.1830062866211, | |
| "logps/rejected": -116.08586120605469, | |
| "loss": 0.0411, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0561954975128174, | |
| "rewards/margins": 4.135585784912109, | |
| "rewards/rejected": -2.079390287399292, | |
| "step": 964 | |
| }, | |
| { | |
| "epoch": 61.46, | |
| "grad_norm": 1.0248548984527588, | |
| "learning_rate": 3.702403118789992e-07, | |
| "logits/chosen": -2.5450968742370605, | |
| "logits/rejected": -2.5582926273345947, | |
| "logps/chosen": -109.03056335449219, | |
| "logps/rejected": -122.5210952758789, | |
| "loss": 0.045, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9161659479141235, | |
| "rewards/margins": 3.8867244720458984, | |
| "rewards/rejected": -1.970558524131775, | |
| "step": 968 | |
| }, | |
| { | |
| "epoch": 61.71, | |
| "grad_norm": 1.595377802848816, | |
| "learning_rate": 3.692483077726842e-07, | |
| "logits/chosen": -2.5482635498046875, | |
| "logits/rejected": -2.537240505218506, | |
| "logps/chosen": -114.62641906738281, | |
| "logps/rejected": -122.08514404296875, | |
| "loss": 0.0641, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9590504169464111, | |
| "rewards/margins": 3.6032297611236572, | |
| "rewards/rejected": -1.644179344177246, | |
| "step": 972 | |
| }, | |
| { | |
| "epoch": 61.97, | |
| "grad_norm": 0.8679869771003723, | |
| "learning_rate": 3.682538670810596e-07, | |
| "logits/chosen": -2.5945587158203125, | |
| "logits/rejected": -2.601808786392212, | |
| "logps/chosen": -106.33365631103516, | |
| "logps/rejected": -115.79863739013672, | |
| "loss": 0.0416, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.9806954860687256, | |
| "rewards/margins": 4.1988372802734375, | |
| "rewards/rejected": -2.218141794204712, | |
| "step": 976 | |
| }, | |
| { | |
| "epoch": 62.22, | |
| "grad_norm": 0.6783718466758728, | |
| "learning_rate": 3.6725701012340383e-07, | |
| "logits/chosen": -2.5552518367767334, | |
| "logits/rejected": -2.5349369049072266, | |
| "logps/chosen": -115.44186401367188, | |
| "logps/rejected": -124.41796112060547, | |
| "loss": 0.0501, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0889174938201904, | |
| "rewards/margins": 4.224672317504883, | |
| "rewards/rejected": -2.1357548236846924, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 62.48, | |
| "grad_norm": 1.1256548166275024, | |
| "learning_rate": 3.6625775726836673e-07, | |
| "logits/chosen": -2.5621018409729004, | |
| "logits/rejected": -2.541167736053467, | |
| "logps/chosen": -112.84183502197266, | |
| "logps/rejected": -120.05206298828125, | |
| "loss": 0.0485, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.1025571823120117, | |
| "rewards/margins": 4.009669303894043, | |
| "rewards/rejected": -1.9071123600006104, | |
| "step": 984 | |
| }, | |
| { | |
| "epoch": 62.73, | |
| "grad_norm": 1.6917866468429565, | |
| "learning_rate": 3.6525612893355314e-07, | |
| "logits/chosen": -2.564279317855835, | |
| "logits/rejected": -2.569469928741455, | |
| "logps/chosen": -109.41111755371094, | |
| "logps/rejected": -116.52373504638672, | |
| "loss": 0.0609, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.962429165840149, | |
| "rewards/margins": 3.8329784870147705, | |
| "rewards/rejected": -1.870548963546753, | |
| "step": 988 | |
| }, | |
| { | |
| "epoch": 62.98, | |
| "grad_norm": 0.7290852069854736, | |
| "learning_rate": 3.6425214558510574e-07, | |
| "logits/chosen": -2.5594987869262695, | |
| "logits/rejected": -2.583799362182617, | |
| "logps/chosen": -109.19231414794922, | |
| "logps/rejected": -115.69007110595703, | |
| "loss": 0.0492, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.7536541223526, | |
| "rewards/margins": 3.967104434967041, | |
| "rewards/rejected": -2.2134504318237305, | |
| "step": 992 | |
| }, | |
| { | |
| "epoch": 63.24, | |
| "grad_norm": 0.9925839304924011, | |
| "learning_rate": 3.6324582773728705e-07, | |
| "logits/chosen": -2.5263614654541016, | |
| "logits/rejected": -2.528245210647583, | |
| "logps/chosen": -111.07906341552734, | |
| "logps/rejected": -120.55313873291016, | |
| "loss": 0.0504, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9851771593093872, | |
| "rewards/margins": 3.987983465194702, | |
| "rewards/rejected": -2.0028064250946045, | |
| "step": 996 | |
| }, | |
| { | |
| "epoch": 63.49, | |
| "grad_norm": 0.9143023490905762, | |
| "learning_rate": 3.622371959520599e-07, | |
| "logits/chosen": -2.602895498275757, | |
| "logits/rejected": -2.588486909866333, | |
| "logps/chosen": -113.23171997070312, | |
| "logps/rejected": -125.9736328125, | |
| "loss": 0.0489, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9089276790618896, | |
| "rewards/margins": 3.9534406661987305, | |
| "rewards/rejected": -2.044512987136841, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 63.49, | |
| "eval_logits/chosen": -2.6193482875823975, | |
| "eval_logits/rejected": -2.669416666030884, | |
| "eval_logps/chosen": -122.88963317871094, | |
| "eval_logps/rejected": -109.84455108642578, | |
| "eval_loss": 0.3812538683414459, | |
| "eval_rewards/accuracies": 0.8214285969734192, | |
| "eval_rewards/chosen": 1.1179710626602173, | |
| "eval_rewards/margins": 2.291536808013916, | |
| "eval_rewards/rejected": -1.1735659837722778, | |
| "eval_runtime": 19.3806, | |
| "eval_samples_per_second": 1.445, | |
| "eval_steps_per_second": 1.445, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 63.75, | |
| "grad_norm": 1.1189945936203003, | |
| "learning_rate": 3.612262708386677e-07, | |
| "logits/chosen": -2.549151659011841, | |
| "logits/rejected": -2.5342297554016113, | |
| "logps/chosen": -108.18368530273438, | |
| "logps/rejected": -117.32633972167969, | |
| "loss": 0.0552, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.9939894676208496, | |
| "rewards/margins": 4.113208293914795, | |
| "rewards/rejected": -2.1192190647125244, | |
| "step": 1004 | |
| }, | |
| { | |
| "epoch": 64.0, | |
| "grad_norm": 0.9281061291694641, | |
| "learning_rate": 3.6021307305321294e-07, | |
| "logits/chosen": -2.524024724960327, | |
| "logits/rejected": -2.5277915000915527, | |
| "logps/chosen": -114.27344512939453, | |
| "logps/rejected": -116.72073364257812, | |
| "loss": 0.0341, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9564348459243774, | |
| "rewards/margins": 4.206700325012207, | |
| "rewards/rejected": -2.250265598297119, | |
| "step": 1008 | |
| }, | |
| { | |
| "epoch": 64.25, | |
| "grad_norm": 0.9672576785087585, | |
| "learning_rate": 3.591976232982355e-07, | |
| "logits/chosen": -2.5507400035858154, | |
| "logits/rejected": -2.5656113624572754, | |
| "logps/chosen": -105.44967651367188, | |
| "logps/rejected": -119.23912048339844, | |
| "loss": 0.0448, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9570220708847046, | |
| "rewards/margins": 4.062612533569336, | |
| "rewards/rejected": -2.105590343475342, | |
| "step": 1012 | |
| }, | |
| { | |
| "epoch": 64.51, | |
| "grad_norm": 0.9295753836631775, | |
| "learning_rate": 3.581799423222894e-07, | |
| "logits/chosen": -2.5452866554260254, | |
| "logits/rejected": -2.5547432899475098, | |
| "logps/chosen": -114.10334777832031, | |
| "logps/rejected": -124.42996215820312, | |
| "loss": 0.04, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8515616655349731, | |
| "rewards/margins": 4.295123100280762, | |
| "rewards/rejected": -2.443561315536499, | |
| "step": 1016 | |
| }, | |
| { | |
| "epoch": 64.76, | |
| "grad_norm": 1.1630864143371582, | |
| "learning_rate": 3.57160050919519e-07, | |
| "logits/chosen": -2.5098414421081543, | |
| "logits/rejected": -2.5110950469970703, | |
| "logps/chosen": -112.0383529663086, | |
| "logps/rejected": -125.51400756835938, | |
| "loss": 0.0388, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0512752532958984, | |
| "rewards/margins": 4.277214050292969, | |
| "rewards/rejected": -2.2259387969970703, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 65.02, | |
| "grad_norm": 1.1040410995483398, | |
| "learning_rate": 3.561379699292338e-07, | |
| "logits/chosen": -2.5779294967651367, | |
| "logits/rejected": -2.532668113708496, | |
| "logps/chosen": -114.37411499023438, | |
| "logps/rejected": -112.6378173828125, | |
| "loss": 0.0557, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 2.033304214477539, | |
| "rewards/margins": 3.9332354068756104, | |
| "rewards/rejected": -1.8999310731887817, | |
| "step": 1024 | |
| }, | |
| { | |
| "epoch": 65.27, | |
| "grad_norm": 0.9148807525634766, | |
| "learning_rate": 3.5511372023548305e-07, | |
| "logits/chosen": -2.5077385902404785, | |
| "logits/rejected": -2.5048418045043945, | |
| "logps/chosen": -114.31764221191406, | |
| "logps/rejected": -126.50031280517578, | |
| "loss": 0.0509, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8525328636169434, | |
| "rewards/margins": 4.272181987762451, | |
| "rewards/rejected": -2.4196486473083496, | |
| "step": 1028 | |
| }, | |
| { | |
| "epoch": 65.52, | |
| "grad_norm": 1.133243441581726, | |
| "learning_rate": 3.5408732276662876e-07, | |
| "logits/chosen": -2.5681333541870117, | |
| "logits/rejected": -2.5366806983947754, | |
| "logps/chosen": -108.5892333984375, | |
| "logps/rejected": -117.7904281616211, | |
| "loss": 0.0557, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.8624019622802734, | |
| "rewards/margins": 3.896911144256592, | |
| "rewards/rejected": -2.0345091819763184, | |
| "step": 1032 | |
| }, | |
| { | |
| "epoch": 65.78, | |
| "grad_norm": 1.1898314952850342, | |
| "learning_rate": 3.530587984949183e-07, | |
| "logits/chosen": -2.5765511989593506, | |
| "logits/rejected": -2.5598666667938232, | |
| "logps/chosen": -116.97229766845703, | |
| "logps/rejected": -121.18170166015625, | |
| "loss": 0.0372, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0075888633728027, | |
| "rewards/margins": 4.278448581695557, | |
| "rewards/rejected": -2.270859718322754, | |
| "step": 1036 | |
| }, | |
| { | |
| "epoch": 66.03, | |
| "grad_norm": 0.6972880363464355, | |
| "learning_rate": 3.5202816843605535e-07, | |
| "logits/chosen": -2.5401930809020996, | |
| "logits/rejected": -2.5507655143737793, | |
| "logps/chosen": -112.83836364746094, | |
| "logps/rejected": -125.02684020996094, | |
| "loss": 0.0293, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9683219194412231, | |
| "rewards/margins": 4.259614944458008, | |
| "rewards/rejected": -2.291292905807495, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 66.03, | |
| "eval_logits/chosen": -2.607311725616455, | |
| "eval_logits/rejected": -2.655458450317383, | |
| "eval_logps/chosen": -123.29393005371094, | |
| "eval_logps/rejected": -111.0044937133789, | |
| "eval_loss": 0.38039258122444153, | |
| "eval_rewards/accuracies": 0.8214285969734192, | |
| "eval_rewards/chosen": 1.0775415897369385, | |
| "eval_rewards/margins": 2.367100954055786, | |
| "eval_rewards/rejected": -1.2895591259002686, | |
| "eval_runtime": 19.3785, | |
| "eval_samples_per_second": 1.445, | |
| "eval_steps_per_second": 1.445, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 66.29, | |
| "grad_norm": 1.1324729919433594, | |
| "learning_rate": 3.5099545364877136e-07, | |
| "logits/chosen": -2.540757417678833, | |
| "logits/rejected": -2.5166916847229004, | |
| "logps/chosen": -107.70198822021484, | |
| "logps/rejected": -126.90846252441406, | |
| "loss": 0.0483, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7595597505569458, | |
| "rewards/margins": 3.971503257751465, | |
| "rewards/rejected": -2.2119436264038086, | |
| "step": 1044 | |
| }, | |
| { | |
| "epoch": 66.54, | |
| "grad_norm": 0.5052275061607361, | |
| "learning_rate": 3.499606752343945e-07, | |
| "logits/chosen": -2.5287301540374756, | |
| "logits/rejected": -2.546327590942383, | |
| "logps/chosen": -113.01952362060547, | |
| "logps/rejected": -116.29533386230469, | |
| "loss": 0.0356, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 2.0772006511688232, | |
| "rewards/margins": 4.4480438232421875, | |
| "rewards/rejected": -2.3708431720733643, | |
| "step": 1048 | |
| }, | |
| { | |
| "epoch": 66.79, | |
| "grad_norm": 0.5736338496208191, | |
| "learning_rate": 3.489238543364187e-07, | |
| "logits/chosen": -2.5480170249938965, | |
| "logits/rejected": -2.528210163116455, | |
| "logps/chosen": -110.82701873779297, | |
| "logps/rejected": -121.10733032226562, | |
| "loss": 0.0264, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.2313339710235596, | |
| "rewards/margins": 4.678577423095703, | |
| "rewards/rejected": -2.4472432136535645, | |
| "step": 1052 | |
| }, | |
| { | |
| "epoch": 67.05, | |
| "grad_norm": 0.7738077044487, | |
| "learning_rate": 3.4788501214007186e-07, | |
| "logits/chosen": -2.5314526557922363, | |
| "logits/rejected": -2.513582468032837, | |
| "logps/chosen": -116.080322265625, | |
| "logps/rejected": -121.7528076171875, | |
| "loss": 0.0543, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8950271606445312, | |
| "rewards/margins": 4.078221797943115, | |
| "rewards/rejected": -2.183194398880005, | |
| "step": 1056 | |
| }, | |
| { | |
| "epoch": 67.3, | |
| "grad_norm": 0.618524432182312, | |
| "learning_rate": 3.468441698718827e-07, | |
| "logits/chosen": -2.559298276901245, | |
| "logits/rejected": -2.551567554473877, | |
| "logps/chosen": -111.18084716796875, | |
| "logps/rejected": -122.01901245117188, | |
| "loss": 0.0331, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9931435585021973, | |
| "rewards/margins": 4.283310413360596, | |
| "rewards/rejected": -2.2901663780212402, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 67.56, | |
| "grad_norm": 0.9859129786491394, | |
| "learning_rate": 3.4580134879924727e-07, | |
| "logits/chosen": -2.5489492416381836, | |
| "logits/rejected": -2.5233051776885986, | |
| "logps/chosen": -114.399658203125, | |
| "logps/rejected": -118.5413818359375, | |
| "loss": 0.0444, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.026573896408081, | |
| "rewards/margins": 4.122701644897461, | |
| "rewards/rejected": -2.096128225326538, | |
| "step": 1064 | |
| }, | |
| { | |
| "epoch": 67.81, | |
| "grad_norm": 0.8273893594741821, | |
| "learning_rate": 3.4475657022999414e-07, | |
| "logits/chosen": -2.5420095920562744, | |
| "logits/rejected": -2.554413318634033, | |
| "logps/chosen": -111.864501953125, | |
| "logps/rejected": -121.59029388427734, | |
| "loss": 0.0398, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.890322208404541, | |
| "rewards/margins": 4.2500786781311035, | |
| "rewards/rejected": -2.3597564697265625, | |
| "step": 1068 | |
| }, | |
| { | |
| "epoch": 68.06, | |
| "grad_norm": 0.8102222084999084, | |
| "learning_rate": 3.4370985551194926e-07, | |
| "logits/chosen": -2.489243984222412, | |
| "logits/rejected": -2.492791175842285, | |
| "logps/chosen": -105.2918701171875, | |
| "logps/rejected": -127.78358459472656, | |
| "loss": 0.0428, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.985128402709961, | |
| "rewards/margins": 4.529362678527832, | |
| "rewards/rejected": -2.544234275817871, | |
| "step": 1072 | |
| }, | |
| { | |
| "epoch": 68.32, | |
| "grad_norm": 0.8171394467353821, | |
| "learning_rate": 3.426612260324995e-07, | |
| "logits/chosen": -2.5788776874542236, | |
| "logits/rejected": -2.5668318271636963, | |
| "logps/chosen": -113.4781265258789, | |
| "logps/rejected": -119.6295166015625, | |
| "loss": 0.031, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9655818939208984, | |
| "rewards/margins": 4.498297214508057, | |
| "rewards/rejected": -2.5327155590057373, | |
| "step": 1076 | |
| }, | |
| { | |
| "epoch": 68.57, | |
| "grad_norm": 0.6531574130058289, | |
| "learning_rate": 3.41610703218156e-07, | |
| "logits/chosen": -2.5451254844665527, | |
| "logits/rejected": -2.539482593536377, | |
| "logps/chosen": -111.55757141113281, | |
| "logps/rejected": -122.69400787353516, | |
| "loss": 0.038, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8812763690948486, | |
| "rewards/margins": 4.159421920776367, | |
| "rewards/rejected": -2.2781457901000977, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 68.57, | |
| "eval_logits/chosen": -2.595308542251587, | |
| "eval_logits/rejected": -2.6415059566497803, | |
| "eval_logps/chosen": -123.70062255859375, | |
| "eval_logps/rejected": -112.08638763427734, | |
| "eval_loss": 0.38112249970436096, | |
| "eval_rewards/accuracies": 0.8214285969734192, | |
| "eval_rewards/chosen": 1.0368707180023193, | |
| "eval_rewards/margins": 2.4346213340759277, | |
| "eval_rewards/rejected": -1.3977504968643188, | |
| "eval_runtime": 19.362, | |
| "eval_samples_per_second": 1.446, | |
| "eval_steps_per_second": 1.446, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 68.83, | |
| "grad_norm": 0.43723320960998535, | |
| "learning_rate": 3.405583085341161e-07, | |
| "logits/chosen": -2.4995832443237305, | |
| "logits/rejected": -2.4867968559265137, | |
| "logps/chosen": -113.4669418334961, | |
| "logps/rejected": -126.17251586914062, | |
| "loss": 0.0472, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.8598037958145142, | |
| "rewards/margins": 4.452800750732422, | |
| "rewards/rejected": -2.5929970741271973, | |
| "step": 1084 | |
| }, | |
| { | |
| "epoch": 69.08, | |
| "grad_norm": 0.6834803819656372, | |
| "learning_rate": 3.3950406348382475e-07, | |
| "logits/chosen": -2.5350327491760254, | |
| "logits/rejected": -2.523555040359497, | |
| "logps/chosen": -109.96227264404297, | |
| "logps/rejected": -127.48822021484375, | |
| "loss": 0.0343, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8542053699493408, | |
| "rewards/margins": 4.349175453186035, | |
| "rewards/rejected": -2.4949703216552734, | |
| "step": 1088 | |
| }, | |
| { | |
| "epoch": 69.33, | |
| "grad_norm": 0.9091951251029968, | |
| "learning_rate": 3.384479896085353e-07, | |
| "logits/chosen": -2.5270514488220215, | |
| "logits/rejected": -2.518280267715454, | |
| "logps/chosen": -112.04927825927734, | |
| "logps/rejected": -118.20771789550781, | |
| "loss": 0.0355, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.085174322128296, | |
| "rewards/margins": 4.528075218200684, | |
| "rewards/rejected": -2.442901134490967, | |
| "step": 1092 | |
| }, | |
| { | |
| "epoch": 69.59, | |
| "grad_norm": 0.7112252116203308, | |
| "learning_rate": 3.3739010848686903e-07, | |
| "logits/chosen": -2.5208559036254883, | |
| "logits/rejected": -2.518951177597046, | |
| "logps/chosen": -114.5992431640625, | |
| "logps/rejected": -129.33493041992188, | |
| "loss": 0.029, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0512499809265137, | |
| "rewards/margins": 4.543766498565674, | |
| "rewards/rejected": -2.492516279220581, | |
| "step": 1096 | |
| }, | |
| { | |
| "epoch": 69.84, | |
| "grad_norm": 0.948445200920105, | |
| "learning_rate": 3.3633044173437487e-07, | |
| "logits/chosen": -2.53946590423584, | |
| "logits/rejected": -2.5132739543914795, | |
| "logps/chosen": -113.12407684326172, | |
| "logps/rejected": -120.42703247070312, | |
| "loss": 0.0376, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8349709510803223, | |
| "rewards/margins": 4.3107500076293945, | |
| "rewards/rejected": -2.4757792949676514, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 70.1, | |
| "grad_norm": 1.1577597856521606, | |
| "learning_rate": 3.352690110030869e-07, | |
| "logits/chosen": -2.5429604053497314, | |
| "logits/rejected": -2.5576508045196533, | |
| "logps/chosen": -107.9112548828125, | |
| "logps/rejected": -126.51514434814453, | |
| "loss": 0.0525, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.8008710145950317, | |
| "rewards/margins": 4.010225772857666, | |
| "rewards/rejected": -2.2093546390533447, | |
| "step": 1104 | |
| }, | |
| { | |
| "epoch": 70.35, | |
| "grad_norm": 0.8540000915527344, | |
| "learning_rate": 3.342058379810825e-07, | |
| "logits/chosen": -2.5239222049713135, | |
| "logits/rejected": -2.508244037628174, | |
| "logps/chosen": -111.51808166503906, | |
| "logps/rejected": -127.96340942382812, | |
| "loss": 0.039, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.8233864307403564, | |
| "rewards/margins": 4.78690242767334, | |
| "rewards/rejected": -2.9635162353515625, | |
| "step": 1108 | |
| }, | |
| { | |
| "epoch": 70.6, | |
| "grad_norm": 0.8324258923530579, | |
| "learning_rate": 3.33140944392039e-07, | |
| "logits/chosen": -2.560696601867676, | |
| "logits/rejected": -2.5394399166107178, | |
| "logps/chosen": -104.02289581298828, | |
| "logps/rejected": -118.75849914550781, | |
| "loss": 0.0335, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.004833698272705, | |
| "rewards/margins": 4.301353931427002, | |
| "rewards/rejected": -2.2965199947357178, | |
| "step": 1112 | |
| }, | |
| { | |
| "epoch": 70.86, | |
| "grad_norm": 0.9126656651496887, | |
| "learning_rate": 3.3207435199479005e-07, | |
| "logits/chosen": -2.495453119277954, | |
| "logits/rejected": -2.476893424987793, | |
| "logps/chosen": -124.06426239013672, | |
| "logps/rejected": -126.9112777709961, | |
| "loss": 0.0275, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0254414081573486, | |
| "rewards/margins": 4.661925315856934, | |
| "rewards/rejected": -2.636483669281006, | |
| "step": 1116 | |
| }, | |
| { | |
| "epoch": 71.11, | |
| "grad_norm": 0.7946174740791321, | |
| "learning_rate": 3.3100608258288067e-07, | |
| "logits/chosen": -2.558216094970703, | |
| "logits/rejected": -2.5525529384613037, | |
| "logps/chosen": -109.46134948730469, | |
| "logps/rejected": -122.28605651855469, | |
| "loss": 0.0378, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9690169095993042, | |
| "rewards/margins": 4.215582370758057, | |
| "rewards/rejected": -2.246565580368042, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 71.11, | |
| "eval_logits/chosen": -2.5836751461029053, | |
| "eval_logits/rejected": -2.6279237270355225, | |
| "eval_logps/chosen": -124.22281646728516, | |
| "eval_logps/rejected": -113.24488067626953, | |
| "eval_loss": 0.3797485828399658, | |
| "eval_rewards/accuracies": 0.8214285969734192, | |
| "eval_rewards/chosen": 0.9846526384353638, | |
| "eval_rewards/margins": 2.4982528686523438, | |
| "eval_rewards/rejected": -1.51360023021698, | |
| "eval_runtime": 19.3727, | |
| "eval_samples_per_second": 1.445, | |
| "eval_steps_per_second": 1.445, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 71.37, | |
| "grad_norm": 0.8299816846847534, | |
| "learning_rate": 3.29936157984122e-07, | |
| "logits/chosen": -2.49894118309021, | |
| "logits/rejected": -2.514620304107666, | |
| "logps/chosen": -113.71720886230469, | |
| "logps/rejected": -125.64193725585938, | |
| "loss": 0.0336, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9318768978118896, | |
| "rewards/margins": 4.486914157867432, | |
| "rewards/rejected": -2.555037021636963, | |
| "step": 1124 | |
| }, | |
| { | |
| "epoch": 71.62, | |
| "grad_norm": 1.118394136428833, | |
| "learning_rate": 3.2886460006014564e-07, | |
| "logits/chosen": -2.5559606552124023, | |
| "logits/rejected": -2.531343936920166, | |
| "logps/chosen": -115.6338119506836, | |
| "logps/rejected": -127.78579711914062, | |
| "loss": 0.0327, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0132534503936768, | |
| "rewards/margins": 4.542009353637695, | |
| "rewards/rejected": -2.5287556648254395, | |
| "step": 1128 | |
| }, | |
| { | |
| "epoch": 71.87, | |
| "grad_norm": 0.7830736041069031, | |
| "learning_rate": 3.2779143070595654e-07, | |
| "logits/chosen": -2.500929594039917, | |
| "logits/rejected": -2.489039897918701, | |
| "logps/chosen": -109.16018676757812, | |
| "logps/rejected": -119.57136535644531, | |
| "loss": 0.0369, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.957329511642456, | |
| "rewards/margins": 4.499155521392822, | |
| "rewards/rejected": -2.541826009750366, | |
| "step": 1132 | |
| }, | |
| { | |
| "epoch": 72.13, | |
| "grad_norm": 0.8632897138595581, | |
| "learning_rate": 3.2671667184948606e-07, | |
| "logits/chosen": -2.499505043029785, | |
| "logits/rejected": -2.479982852935791, | |
| "logps/chosen": -113.5299072265625, | |
| "logps/rejected": -124.4341049194336, | |
| "loss": 0.0272, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7402851581573486, | |
| "rewards/margins": 4.627835750579834, | |
| "rewards/rejected": -2.8875510692596436, | |
| "step": 1136 | |
| }, | |
| { | |
| "epoch": 72.38, | |
| "grad_norm": 0.5583193302154541, | |
| "learning_rate": 3.2564034545114307e-07, | |
| "logits/chosen": -2.5371761322021484, | |
| "logits/rejected": -2.5350067615509033, | |
| "logps/chosen": -113.26782989501953, | |
| "logps/rejected": -123.95820617675781, | |
| "loss": 0.0311, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.9537227153778076, | |
| "rewards/margins": 4.536089897155762, | |
| "rewards/rejected": -2.582366943359375, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 72.63, | |
| "grad_norm": 0.7022975087165833, | |
| "learning_rate": 3.2456247350336644e-07, | |
| "logits/chosen": -2.4975104331970215, | |
| "logits/rejected": -2.4862759113311768, | |
| "logps/chosen": -116.6255111694336, | |
| "logps/rejected": -124.94264221191406, | |
| "loss": 0.0405, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.032803535461426, | |
| "rewards/margins": 4.609248161315918, | |
| "rewards/rejected": -2.576444387435913, | |
| "step": 1144 | |
| }, | |
| { | |
| "epoch": 72.89, | |
| "grad_norm": 0.6374495625495911, | |
| "learning_rate": 3.234830780301749e-07, | |
| "logits/chosen": -2.5176424980163574, | |
| "logits/rejected": -2.5058205127716064, | |
| "logps/chosen": -103.94803619384766, | |
| "logps/rejected": -128.84860229492188, | |
| "loss": 0.0329, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.949141502380371, | |
| "rewards/margins": 4.762382507324219, | |
| "rewards/rejected": -2.8132412433624268, | |
| "step": 1148 | |
| }, | |
| { | |
| "epoch": 73.14, | |
| "grad_norm": 0.6864801049232483, | |
| "learning_rate": 3.224021810867168e-07, | |
| "logits/chosen": -2.519915819168091, | |
| "logits/rejected": -2.5309557914733887, | |
| "logps/chosen": -112.75774383544922, | |
| "logps/rejected": -125.47956085205078, | |
| "loss": 0.0333, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8436543941497803, | |
| "rewards/margins": 4.395434856414795, | |
| "rewards/rejected": -2.5517804622650146, | |
| "step": 1152 | |
| }, | |
| { | |
| "epoch": 73.4, | |
| "grad_norm": 0.8171544075012207, | |
| "learning_rate": 3.213198047588205e-07, | |
| "logits/chosen": -2.529101610183716, | |
| "logits/rejected": -2.506753921508789, | |
| "logps/chosen": -111.0318603515625, | |
| "logps/rejected": -128.049072265625, | |
| "loss": 0.0335, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7638250589370728, | |
| "rewards/margins": 4.518660068511963, | |
| "rewards/rejected": -2.7548348903656006, | |
| "step": 1156 | |
| }, | |
| { | |
| "epoch": 73.65, | |
| "grad_norm": 0.7904006838798523, | |
| "learning_rate": 3.202359711625417e-07, | |
| "logits/chosen": -2.507455348968506, | |
| "logits/rejected": -2.48976993560791, | |
| "logps/chosen": -113.42962646484375, | |
| "logps/rejected": -128.28025817871094, | |
| "loss": 0.0288, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.8584176301956177, | |
| "rewards/margins": 4.5566816329956055, | |
| "rewards/rejected": -2.6982643604278564, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 73.65, | |
| "eval_logits/chosen": -2.5730206966400146, | |
| "eval_logits/rejected": -2.615673303604126, | |
| "eval_logps/chosen": -124.67210388183594, | |
| "eval_logps/rejected": -114.29837799072266, | |
| "eval_loss": 0.38018837571144104, | |
| "eval_rewards/accuracies": 0.8214285969734192, | |
| "eval_rewards/chosen": 0.939724326133728, | |
| "eval_rewards/margins": 2.558673143386841, | |
| "eval_rewards/rejected": -1.6189485788345337, | |
| "eval_runtime": 19.4065, | |
| "eval_samples_per_second": 1.443, | |
| "eval_steps_per_second": 1.443, | |
| "step": 1160 | |
| } | |
| ], | |
| "logging_steps": 4, | |
| "max_steps": 2800, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 187, | |
| "save_steps": 40, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |