diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,3999 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 100, + "global_step": 1250, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0008, + "grad_norm": 1.3400232791900635, + "learning_rate": 4e-08, + "logits/chosen": -2.951728105545044, + "logits/rejected": -3.0115513801574707, + "logps/chosen": -261.5080261230469, + "logps/rejected": -337.26708984375, + "loss": 0.6931, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "epoch": 0.004, + "grad_norm": 1.3120373487472534, + "learning_rate": 2.0000000000000002e-07, + "logits/chosen": -2.892902374267578, + "logits/rejected": -2.8663315773010254, + "logps/chosen": -327.0978088378906, + "logps/rejected": -271.5657043457031, + "loss": 0.6929, + "rewards/accuracies": 0.421875, + "rewards/chosen": 0.0005052188062109053, + "rewards/margins": 0.0005590975051745772, + "rewards/rejected": -5.3878684411756694e-05, + "step": 5 + }, + { + "epoch": 0.008, + "grad_norm": 1.4112803936004639, + "learning_rate": 4.0000000000000003e-07, + "logits/chosen": -2.8438823223114014, + "logits/rejected": -2.8227336406707764, + "logps/chosen": -278.82293701171875, + "logps/rejected": -225.83895874023438, + "loss": 0.693, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.000398232601583004, + "rewards/margins": 0.0003281077661085874, + "rewards/rejected": -0.0007263403385877609, + "step": 10 + }, + { + "epoch": 0.012, + "grad_norm": 1.446423888206482, + "learning_rate": 6.000000000000001e-07, + "logits/chosen": -2.9413020610809326, + "logits/rejected": -2.9195456504821777, + "logps/chosen": -338.17449951171875, + "logps/rejected": -264.4447021484375, + "loss": 0.6932, + "rewards/accuracies": 0.4625000059604645, + "rewards/chosen": -0.00040648109279572964, + "rewards/margins": -0.00015654772869311273, + "rewards/rejected": -0.0002499335096217692, + "step": 15 + }, + { + "epoch": 0.016, + "grad_norm": 1.2574288845062256, + "learning_rate": 8.000000000000001e-07, + "logits/chosen": -2.8474764823913574, + "logits/rejected": -2.80472993850708, + "logps/chosen": -284.499755859375, + "logps/rejected": -265.3078918457031, + "loss": 0.6929, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.00043911076500080526, + "rewards/margins": 0.000450963998446241, + "rewards/rejected": -1.1853216165036429e-05, + "step": 20 + }, + { + "epoch": 0.02, + "grad_norm": 1.0634217262268066, + "learning_rate": 1.0000000000000002e-06, + "logits/chosen": -2.9212899208068848, + "logits/rejected": -2.8857905864715576, + "logps/chosen": -282.70477294921875, + "logps/rejected": -250.54092407226562, + "loss": 0.6931, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 4.3422729504527524e-05, + "rewards/margins": 0.0001133212135755457, + "rewards/rejected": -6.989858957240358e-05, + "step": 25 + }, + { + "epoch": 0.024, + "grad_norm": 1.285556435585022, + "learning_rate": 1.2000000000000002e-06, + "logits/chosen": -2.8694255352020264, + "logits/rejected": -2.821131944656372, + "logps/chosen": -248.3980255126953, + "logps/rejected": -239.7887420654297, + "loss": 0.693, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": 0.0004804051131941378, + "rewards/margins": 0.0003015303227584809, + "rewards/rejected": 0.00017887470312416553, + "step": 30 + }, + { + "epoch": 0.028, + "grad_norm": 1.469935417175293, + "learning_rate": 1.4000000000000001e-06, + "logits/chosen": -2.8213143348693848, + "logits/rejected": -2.830141305923462, + "logps/chosen": -260.638916015625, + "logps/rejected": -252.28701782226562, + "loss": 0.6935, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.0009641313808970153, + "rewards/margins": -0.0007172044133767486, + "rewards/rejected": -0.00024692711303941905, + "step": 35 + }, + { + "epoch": 0.032, + "grad_norm": 1.6270555257797241, + "learning_rate": 1.6000000000000001e-06, + "logits/chosen": -2.841813802719116, + "logits/rejected": -2.8207640647888184, + "logps/chosen": -225.63058471679688, + "logps/rejected": -254.8463592529297, + "loss": 0.6926, + "rewards/accuracies": 0.5625, + "rewards/chosen": 5.770945062977262e-05, + "rewards/margins": 0.0010281356517225504, + "rewards/rejected": -0.0009704261319711804, + "step": 40 + }, + { + "epoch": 0.036, + "grad_norm": 1.2644410133361816, + "learning_rate": 1.8000000000000001e-06, + "logits/chosen": -2.8922643661499023, + "logits/rejected": -2.8955490589141846, + "logps/chosen": -262.701416015625, + "logps/rejected": -257.9744567871094, + "loss": 0.6925, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.0004659576225094497, + "rewards/margins": 0.0013163817347958684, + "rewards/rejected": -0.0008504241704940796, + "step": 45 + }, + { + "epoch": 0.04, + "grad_norm": 1.38533353805542, + "learning_rate": 2.0000000000000003e-06, + "logits/chosen": -2.8122503757476807, + "logits/rejected": -2.768099069595337, + "logps/chosen": -246.9253387451172, + "logps/rejected": -221.22207641601562, + "loss": 0.6929, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": 0.00014323795039672405, + "rewards/margins": 0.0004161189717706293, + "rewards/rejected": -0.00027288100682199, + "step": 50 + }, + { + "epoch": 0.044, + "grad_norm": 1.1605815887451172, + "learning_rate": 2.2e-06, + "logits/chosen": -2.8503968715667725, + "logits/rejected": -2.8304195404052734, + "logps/chosen": -289.88092041015625, + "logps/rejected": -304.974609375, + "loss": 0.6933, + "rewards/accuracies": 0.4625000059604645, + "rewards/chosen": -0.00040247183642350137, + "rewards/margins": -0.00022780350991524756, + "rewards/rejected": -0.00017466834106016904, + "step": 55 + }, + { + "epoch": 0.048, + "grad_norm": 1.273954153060913, + "learning_rate": 2.4000000000000003e-06, + "logits/chosen": -2.89465594291687, + "logits/rejected": -2.875718832015991, + "logps/chosen": -265.1037902832031, + "logps/rejected": -274.2173767089844, + "loss": 0.6924, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": 0.0005437713698484004, + "rewards/margins": 0.001443098415620625, + "rewards/rejected": -0.0008993271621875465, + "step": 60 + }, + { + "epoch": 0.052, + "grad_norm": 1.0502583980560303, + "learning_rate": 2.6e-06, + "logits/chosen": -2.862156391143799, + "logits/rejected": -2.837881565093994, + "logps/chosen": -242.22341918945312, + "logps/rejected": -258.85699462890625, + "loss": 0.6923, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.000915366574190557, + "rewards/margins": 0.001800536410883069, + "rewards/rejected": -0.0008851696038618684, + "step": 65 + }, + { + "epoch": 0.056, + "grad_norm": 1.203967571258545, + "learning_rate": 2.8000000000000003e-06, + "logits/chosen": -2.8611202239990234, + "logits/rejected": -2.8584847450256348, + "logps/chosen": -256.2646179199219, + "logps/rejected": -239.5492401123047, + "loss": 0.692, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.0021997836884111166, + "rewards/margins": 0.0023759277537465096, + "rewards/rejected": -0.00017614415264688432, + "step": 70 + }, + { + "epoch": 0.06, + "grad_norm": 1.4282208681106567, + "learning_rate": 3e-06, + "logits/chosen": -2.921938180923462, + "logits/rejected": -2.8600423336029053, + "logps/chosen": -286.9979553222656, + "logps/rejected": -258.0650939941406, + "loss": 0.6917, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": 0.0020814109593629837, + "rewards/margins": 0.0028700605034828186, + "rewards/rejected": -0.0007886493694968522, + "step": 75 + }, + { + "epoch": 0.064, + "grad_norm": 1.4035444259643555, + "learning_rate": 3.2000000000000003e-06, + "logits/chosen": -2.8771731853485107, + "logits/rejected": -2.8718912601470947, + "logps/chosen": -257.95562744140625, + "logps/rejected": -248.6924285888672, + "loss": 0.6916, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.004527095705270767, + "rewards/margins": 0.003182282205671072, + "rewards/rejected": 0.0013448137324303389, + "step": 80 + }, + { + "epoch": 0.068, + "grad_norm": 1.4235211610794067, + "learning_rate": 3.4000000000000005e-06, + "logits/chosen": -2.9201653003692627, + "logits/rejected": -2.8719587326049805, + "logps/chosen": -307.7834777832031, + "logps/rejected": -268.5202941894531, + "loss": 0.6914, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": 0.005866583436727524, + "rewards/margins": 0.0034526665695011616, + "rewards/rejected": 0.0024139168672263622, + "step": 85 + }, + { + "epoch": 0.072, + "grad_norm": 1.1721140146255493, + "learning_rate": 3.6000000000000003e-06, + "logits/chosen": -2.8801143169403076, + "logits/rejected": -2.844945192337036, + "logps/chosen": -238.00100708007812, + "logps/rejected": -238.72006225585938, + "loss": 0.6919, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.005659895483404398, + "rewards/margins": 0.0026023960672318935, + "rewards/rejected": 0.003057498950511217, + "step": 90 + }, + { + "epoch": 0.076, + "grad_norm": 1.2834060192108154, + "learning_rate": 3.8000000000000005e-06, + "logits/chosen": -2.896768093109131, + "logits/rejected": -2.9003615379333496, + "logps/chosen": -268.2308654785156, + "logps/rejected": -241.99667358398438, + "loss": 0.6904, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.0076117864809930325, + "rewards/margins": 0.005503328982740641, + "rewards/rejected": 0.0021084570325911045, + "step": 95 + }, + { + "epoch": 0.08, + "grad_norm": 1.233703374862671, + "learning_rate": 4.000000000000001e-06, + "logits/chosen": -2.8716769218444824, + "logits/rejected": -2.8608012199401855, + "logps/chosen": -292.55133056640625, + "logps/rejected": -255.74276733398438, + "loss": 0.6895, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.009478461928665638, + "rewards/margins": 0.007306996732950211, + "rewards/rejected": 0.00217146473005414, + "step": 100 + }, + { + "epoch": 0.08, + "eval_logits/chosen": -2.890129566192627, + "eval_logits/rejected": -2.8480653762817383, + "eval_logps/chosen": -282.2447204589844, + "eval_logps/rejected": -247.7537384033203, + "eval_loss": 0.6896404027938843, + "eval_rewards/accuracies": 0.6626983880996704, + "eval_rewards/chosen": 0.00993373617529869, + "eval_rewards/margins": 0.007174656726419926, + "eval_rewards/rejected": 0.0027590803802013397, + "eval_runtime": 167.6825, + "eval_samples_per_second": 2.982, + "eval_steps_per_second": 0.376, + "step": 100 + }, + { + "epoch": 0.084, + "grad_norm": 1.3204169273376465, + "learning_rate": 4.2000000000000004e-06, + "logits/chosen": -2.8663666248321533, + "logits/rejected": -2.820469856262207, + "logps/chosen": -272.216552734375, + "logps/rejected": -261.4291076660156, + "loss": 0.6905, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.008843997493386269, + "rewards/margins": 0.005469636060297489, + "rewards/rejected": 0.003374360501766205, + "step": 105 + }, + { + "epoch": 0.088, + "grad_norm": 1.2558304071426392, + "learning_rate": 4.4e-06, + "logits/chosen": -2.935101270675659, + "logits/rejected": -2.903439521789551, + "logps/chosen": -251.76089477539062, + "logps/rejected": -246.2103729248047, + "loss": 0.6905, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": 0.010050037875771523, + "rewards/margins": 0.00540496688336134, + "rewards/rejected": 0.004645070992410183, + "step": 110 + }, + { + "epoch": 0.092, + "grad_norm": 2.3479883670806885, + "learning_rate": 4.600000000000001e-06, + "logits/chosen": -2.8293395042419434, + "logits/rejected": -2.8186140060424805, + "logps/chosen": -225.02096557617188, + "logps/rejected": -294.81719970703125, + "loss": 0.6865, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.012401686981320381, + "rewards/margins": 0.01355154998600483, + "rewards/rejected": -0.0011498630046844482, + "step": 115 + }, + { + "epoch": 0.096, + "grad_norm": 3.1538350582122803, + "learning_rate": 4.800000000000001e-06, + "logits/chosen": -2.7399497032165527, + "logits/rejected": -2.749803066253662, + "logps/chosen": -275.8015441894531, + "logps/rejected": -249.06002807617188, + "loss": 0.6888, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.01128525473177433, + "rewards/margins": 0.009098999202251434, + "rewards/rejected": 0.002186256693676114, + "step": 120 + }, + { + "epoch": 0.1, + "grad_norm": 1.5341688394546509, + "learning_rate": 5e-06, + "logits/chosen": -2.9408421516418457, + "logits/rejected": -2.924834728240967, + "logps/chosen": -310.1289367675781, + "logps/rejected": -280.08770751953125, + "loss": 0.6868, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.012295490130782127, + "rewards/margins": 0.01287610363215208, + "rewards/rejected": -0.0005806152475997806, + "step": 125 + }, + { + "epoch": 0.104, + "grad_norm": 1.4902671575546265, + "learning_rate": 4.999756310023261e-06, + "logits/chosen": -2.889176845550537, + "logits/rejected": -2.896610975265503, + "logps/chosen": -286.29925537109375, + "logps/rejected": -309.28265380859375, + "loss": 0.6874, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.00844976119697094, + "rewards/margins": 0.011819533072412014, + "rewards/rejected": -0.0033697723411023617, + "step": 130 + }, + { + "epoch": 0.108, + "grad_norm": 1.4190293550491333, + "learning_rate": 4.999025287600886e-06, + "logits/chosen": -2.892551898956299, + "logits/rejected": -2.908194065093994, + "logps/chosen": -274.4278869628906, + "logps/rejected": -265.5429382324219, + "loss": 0.6806, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 0.012322185561060905, + "rewards/margins": 0.025636225938796997, + "rewards/rejected": -0.013314038515090942, + "step": 135 + }, + { + "epoch": 0.112, + "grad_norm": 1.3311868906021118, + "learning_rate": 4.997807075247147e-06, + "logits/chosen": -2.8810911178588867, + "logits/rejected": -2.859647750854492, + "logps/chosen": -247.0091094970703, + "logps/rejected": -236.913818359375, + "loss": 0.6888, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.004415545146912336, + "rewards/margins": 0.009412359446287155, + "rewards/rejected": -0.013827905058860779, + "step": 140 + }, + { + "epoch": 0.116, + "grad_norm": 1.286658525466919, + "learning_rate": 4.996101910454953e-06, + "logits/chosen": -2.9036033153533936, + "logits/rejected": -2.8598740100860596, + "logps/chosen": -273.776123046875, + "logps/rejected": -244.0740509033203, + "loss": 0.6801, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": 0.0010833339765667915, + "rewards/margins": 0.026883091777563095, + "rewards/rejected": -0.02579975686967373, + "step": 145 + }, + { + "epoch": 0.12, + "grad_norm": 1.6749712228775024, + "learning_rate": 4.993910125649561e-06, + "logits/chosen": -2.891787528991699, + "logits/rejected": -2.856782913208008, + "logps/chosen": -293.8242492675781, + "logps/rejected": -247.73068237304688, + "loss": 0.6806, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.010322624817490578, + "rewards/margins": 0.025890743359923363, + "rewards/rejected": -0.015568114817142487, + "step": 150 + }, + { + "epoch": 0.124, + "grad_norm": 1.5311990976333618, + "learning_rate": 4.9912321481237616e-06, + "logits/chosen": -2.7779576778411865, + "logits/rejected": -2.774000883102417, + "logps/chosen": -231.47427368164062, + "logps/rejected": -290.8375549316406, + "loss": 0.6839, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.0004987965803593397, + "rewards/margins": 0.019459182396531105, + "rewards/rejected": -0.018960384652018547, + "step": 155 + }, + { + "epoch": 0.128, + "grad_norm": 1.5341625213623047, + "learning_rate": 4.988068499954578e-06, + "logits/chosen": -2.890897512435913, + "logits/rejected": -2.889413356781006, + "logps/chosen": -316.82025146484375, + "logps/rejected": -312.2231140136719, + "loss": 0.6716, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.031246433034539223, + "rewards/margins": 0.045340072363615036, + "rewards/rejected": -0.014093644917011261, + "step": 160 + }, + { + "epoch": 0.132, + "grad_norm": 1.6371203660964966, + "learning_rate": 4.984419797901491e-06, + "logits/chosen": -2.9241220951080322, + "logits/rejected": -2.9127113819122314, + "logps/chosen": -311.5802001953125, + "logps/rejected": -282.0766906738281, + "loss": 0.6702, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.03161744773387909, + "rewards/margins": 0.04772466421127319, + "rewards/rejected": -0.016107218340039253, + "step": 165 + }, + { + "epoch": 0.136, + "grad_norm": 1.5368744134902954, + "learning_rate": 4.980286753286196e-06, + "logits/chosen": -2.917865514755249, + "logits/rejected": -2.9101455211639404, + "logps/chosen": -275.5201110839844, + "logps/rejected": -273.40301513671875, + "loss": 0.6758, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": 0.02875286340713501, + "rewards/margins": 0.03745796158909798, + "rewards/rejected": -0.008705099113285542, + "step": 170 + }, + { + "epoch": 0.14, + "grad_norm": 1.5606811046600342, + "learning_rate": 4.975670171853926e-06, + "logits/chosen": -2.881425380706787, + "logits/rejected": -2.82181978225708, + "logps/chosen": -268.76611328125, + "logps/rejected": -241.20675659179688, + "loss": 0.6724, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": 0.01504159439355135, + "rewards/margins": 0.04509888216853142, + "rewards/rejected": -0.030057286843657494, + "step": 175 + }, + { + "epoch": 0.144, + "grad_norm": 1.5441210269927979, + "learning_rate": 4.970570953616383e-06, + "logits/chosen": -2.871035099029541, + "logits/rejected": -2.8470711708068848, + "logps/chosen": -271.70977783203125, + "logps/rejected": -250.18887329101562, + "loss": 0.6577, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.02740972302854061, + "rewards/margins": 0.07525759935379028, + "rewards/rejected": -0.04784787446260452, + "step": 180 + }, + { + "epoch": 0.148, + "grad_norm": 1.7091320753097534, + "learning_rate": 4.964990092676263e-06, + "logits/chosen": -2.827620506286621, + "logits/rejected": -2.824796676635742, + "logps/chosen": -272.52923583984375, + "logps/rejected": -226.22506713867188, + "loss": 0.6767, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -0.0010557698551565409, + "rewards/margins": 0.03629080206155777, + "rewards/rejected": -0.03734657168388367, + "step": 185 + }, + { + "epoch": 0.152, + "grad_norm": 1.8644754886627197, + "learning_rate": 4.958928677033465e-06, + "logits/chosen": -2.8307862281799316, + "logits/rejected": -2.8190550804138184, + "logps/chosen": -276.7012023925781, + "logps/rejected": -289.3759765625, + "loss": 0.6642, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": 0.007176427636295557, + "rewards/margins": 0.06312907487154007, + "rewards/rejected": -0.05595264956355095, + "step": 190 + }, + { + "epoch": 0.156, + "grad_norm": 1.796454906463623, + "learning_rate": 4.9523878883729794e-06, + "logits/chosen": -2.8766260147094727, + "logits/rejected": -2.8518166542053223, + "logps/chosen": -288.412841796875, + "logps/rejected": -255.20870971679688, + "loss": 0.656, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.004013822879642248, + "rewards/margins": 0.0801478773355484, + "rewards/rejected": -0.07613405585289001, + "step": 195 + }, + { + "epoch": 0.16, + "grad_norm": 1.861384391784668, + "learning_rate": 4.9453690018345144e-06, + "logits/chosen": -2.8510830402374268, + "logits/rejected": -2.8279526233673096, + "logps/chosen": -255.71249389648438, + "logps/rejected": -257.49163818359375, + "loss": 0.653, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.003837744938209653, + "rewards/margins": 0.08905676007270813, + "rewards/rejected": -0.09289450943470001, + "step": 200 + }, + { + "epoch": 0.16, + "eval_logits/chosen": -2.874197244644165, + "eval_logits/rejected": -2.8339242935180664, + "eval_logps/chosen": -284.5635070800781, + "eval_logps/rejected": -257.5692443847656, + "eval_loss": 0.6569300293922424, + "eval_rewards/accuracies": 0.6865079402923584, + "eval_rewards/chosen": -0.013254065066576004, + "eval_rewards/margins": 0.08214230835437775, + "eval_rewards/rejected": -0.09539636969566345, + "eval_runtime": 166.8148, + "eval_samples_per_second": 2.997, + "eval_steps_per_second": 0.378, + "step": 200 + }, + { + "epoch": 0.164, + "grad_norm": 1.9093883037567139, + "learning_rate": 4.937873385763909e-06, + "logits/chosen": -2.862431764602661, + "logits/rejected": -2.829721689224243, + "logps/chosen": -287.107177734375, + "logps/rejected": -284.347412109375, + "loss": 0.6582, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.03399428352713585, + "rewards/margins": 0.07983867824077606, + "rewards/rejected": -0.1138329729437828, + "step": 205 + }, + { + "epoch": 0.168, + "grad_norm": 1.8923628330230713, + "learning_rate": 4.9299025014463665e-06, + "logits/chosen": -2.8810172080993652, + "logits/rejected": -2.8641536235809326, + "logps/chosen": -248.9971466064453, + "logps/rejected": -245.23355102539062, + "loss": 0.6711, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.046056050807237625, + "rewards/margins": 0.05324975773692131, + "rewards/rejected": -0.09930581599473953, + "step": 210 + }, + { + "epoch": 0.172, + "grad_norm": 1.887384057044983, + "learning_rate": 4.921457902821578e-06, + "logits/chosen": -2.8637945652008057, + "logits/rejected": -2.8070335388183594, + "logps/chosen": -316.33355712890625, + "logps/rejected": -285.98590087890625, + "loss": 0.6668, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.018491100519895554, + "rewards/margins": 0.06612573564052582, + "rewards/rejected": -0.08461683988571167, + "step": 215 + }, + { + "epoch": 0.176, + "grad_norm": 2.3003458976745605, + "learning_rate": 4.912541236180779e-06, + "logits/chosen": -2.7972946166992188, + "logits/rejected": -2.7600889205932617, + "logps/chosen": -325.50244140625, + "logps/rejected": -316.9506530761719, + "loss": 0.6418, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.016769036650657654, + "rewards/margins": 0.11797495931386948, + "rewards/rejected": -0.13474401831626892, + "step": 220 + }, + { + "epoch": 0.18, + "grad_norm": 1.6942790746688843, + "learning_rate": 4.903154239845798e-06, + "logits/chosen": -2.8859641551971436, + "logits/rejected": -2.8270339965820312, + "logps/chosen": -271.8771667480469, + "logps/rejected": -247.03146362304688, + "loss": 0.6483, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.06943339854478836, + "rewards/margins": 0.10388622432947159, + "rewards/rejected": -0.17331962287425995, + "step": 225 + }, + { + "epoch": 0.184, + "grad_norm": 2.1547300815582275, + "learning_rate": 4.893298743830168e-06, + "logits/chosen": -2.793713092803955, + "logits/rejected": -2.806272029876709, + "logps/chosen": -302.48150634765625, + "logps/rejected": -302.66973876953125, + "loss": 0.6308, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.04446331784129143, + "rewards/margins": 0.1429525464773178, + "rewards/rejected": -0.18741586804389954, + "step": 230 + }, + { + "epoch": 0.188, + "grad_norm": 2.574212074279785, + "learning_rate": 4.882976669482368e-06, + "logits/chosen": -2.8080554008483887, + "logits/rejected": -2.778135061264038, + "logps/chosen": -275.0320129394531, + "logps/rejected": -280.06610107421875, + "loss": 0.6431, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.09150179475545883, + "rewards/margins": 0.12502221763134003, + "rewards/rejected": -0.21652403473854065, + "step": 235 + }, + { + "epoch": 0.192, + "grad_norm": 2.8856008052825928, + "learning_rate": 4.8721900291112415e-06, + "logits/chosen": -2.8577849864959717, + "logits/rejected": -2.836282253265381, + "logps/chosen": -291.05352783203125, + "logps/rejected": -275.5093078613281, + "loss": 0.6433, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.016148822382092476, + "rewards/margins": 0.11701644957065582, + "rewards/rejected": -0.13316525518894196, + "step": 240 + }, + { + "epoch": 0.196, + "grad_norm": 2.3244190216064453, + "learning_rate": 4.860940925593703e-06, + "logits/chosen": -2.8803956508636475, + "logits/rejected": -2.8489887714385986, + "logps/chosen": -288.46856689453125, + "logps/rejected": -274.30364990234375, + "loss": 0.637, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": 0.012815827503800392, + "rewards/margins": 0.13950756192207336, + "rewards/rejected": -0.12669174373149872, + "step": 245 + }, + { + "epoch": 0.2, + "grad_norm": 2.1376349925994873, + "learning_rate": 4.849231551964771e-06, + "logits/chosen": -2.8547749519348145, + "logits/rejected": -2.8269331455230713, + "logps/chosen": -254.5790557861328, + "logps/rejected": -242.4806365966797, + "loss": 0.6555, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.016418656334280968, + "rewards/margins": 0.09427244961261749, + "rewards/rejected": -0.11069109290838242, + "step": 250 + }, + { + "epoch": 0.204, + "grad_norm": 3.7023279666900635, + "learning_rate": 4.837064190990036e-06, + "logits/chosen": -2.7928366661071777, + "logits/rejected": -2.8065311908721924, + "logps/chosen": -286.6212158203125, + "logps/rejected": -284.4375, + "loss": 0.6467, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.05689004808664322, + "rewards/margins": 0.11735578626394272, + "rewards/rejected": -0.17424583435058594, + "step": 255 + }, + { + "epoch": 0.208, + "grad_norm": 3.1189470291137695, + "learning_rate": 4.824441214720629e-06, + "logits/chosen": -2.826845169067383, + "logits/rejected": -2.839430332183838, + "logps/chosen": -330.69378662109375, + "logps/rejected": -295.32781982421875, + "loss": 0.6619, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11839203536510468, + "rewards/margins": 0.09098449349403381, + "rewards/rejected": -0.2093765288591385, + "step": 260 + }, + { + "epoch": 0.212, + "grad_norm": 5.202083110809326, + "learning_rate": 4.811365084030784e-06, + "logits/chosen": -2.7906274795532227, + "logits/rejected": -2.7402591705322266, + "logps/chosen": -240.4190673828125, + "logps/rejected": -258.24639892578125, + "loss": 0.6288, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13684885203838348, + "rewards/margins": 0.1477588713169098, + "rewards/rejected": -0.28460773825645447, + "step": 265 + }, + { + "epoch": 0.216, + "grad_norm": 4.058558940887451, + "learning_rate": 4.7978383481380865e-06, + "logits/chosen": -2.8255257606506348, + "logits/rejected": -2.827247381210327, + "logps/chosen": -284.66058349609375, + "logps/rejected": -326.15008544921875, + "loss": 0.6278, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1267203986644745, + "rewards/margins": 0.1748858392238617, + "rewards/rejected": -0.3016062378883362, + "step": 270 + }, + { + "epoch": 0.22, + "grad_norm": 2.6630849838256836, + "learning_rate": 4.783863644106502e-06, + "logits/chosen": -2.8801255226135254, + "logits/rejected": -2.8717246055603027, + "logps/chosen": -279.7027893066406, + "logps/rejected": -273.7590026855469, + "loss": 0.6354, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -0.12922950088977814, + "rewards/margins": 0.1460111439228058, + "rewards/rejected": -0.2752406597137451, + "step": 275 + }, + { + "epoch": 0.224, + "grad_norm": 3.321779251098633, + "learning_rate": 4.769443696332272e-06, + "logits/chosen": -2.8768434524536133, + "logits/rejected": -2.8461215496063232, + "logps/chosen": -292.112548828125, + "logps/rejected": -293.89886474609375, + "loss": 0.6301, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08245956897735596, + "rewards/margins": 0.16300079226493835, + "rewards/rejected": -0.24546034634113312, + "step": 280 + }, + { + "epoch": 0.228, + "grad_norm": 2.9796993732452393, + "learning_rate": 4.754581316012785e-06, + "logits/chosen": -2.875678300857544, + "logits/rejected": -2.8014907836914062, + "logps/chosen": -321.65411376953125, + "logps/rejected": -297.4300842285156, + "loss": 0.6025, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.09748764336109161, + "rewards/margins": 0.22937169671058655, + "rewards/rejected": -0.32685935497283936, + "step": 285 + }, + { + "epoch": 0.232, + "grad_norm": 3.133993148803711, + "learning_rate": 4.7392794005985324e-06, + "logits/chosen": -2.8018627166748047, + "logits/rejected": -2.7942662239074707, + "logps/chosen": -292.52667236328125, + "logps/rejected": -271.00030517578125, + "loss": 0.5949, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.21974220871925354, + "rewards/margins": 0.23409290611743927, + "rewards/rejected": -0.4538350999355316, + "step": 290 + }, + { + "epoch": 0.236, + "grad_norm": 3.11313796043396, + "learning_rate": 4.723540933228245e-06, + "logits/chosen": -2.820415496826172, + "logits/rejected": -2.796581745147705, + "logps/chosen": -329.7557678222656, + "logps/rejected": -321.95233154296875, + "loss": 0.6621, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.3443993926048279, + "rewards/margins": 0.10115940868854523, + "rewards/rejected": -0.4455588757991791, + "step": 295 + }, + { + "epoch": 0.24, + "grad_norm": 3.609873056411743, + "learning_rate": 4.707368982147318e-06, + "logits/chosen": -2.8759727478027344, + "logits/rejected": -2.825862407684326, + "logps/chosen": -333.0445556640625, + "logps/rejected": -285.5638122558594, + "loss": 0.6385, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.3127234876155853, + "rewards/margins": 0.1507173478603363, + "rewards/rejected": -0.46344083547592163, + "step": 300 + }, + { + "epoch": 0.24, + "eval_logits/chosen": -2.8398690223693848, + "eval_logits/rejected": -2.803138494491577, + "eval_logps/chosen": -310.65655517578125, + "eval_logps/rejected": -295.55364990234375, + "eval_loss": 0.6190334558486938, + "eval_rewards/accuracies": 0.6904761791229248, + "eval_rewards/chosen": -0.27418458461761475, + "eval_rewards/margins": 0.20105606317520142, + "eval_rewards/rejected": -0.47524064779281616, + "eval_runtime": 167.0651, + "eval_samples_per_second": 2.993, + "eval_steps_per_second": 0.377, + "step": 300 + }, + { + "epoch": 0.244, + "grad_norm": 5.130188941955566, + "learning_rate": 4.690766700109659e-06, + "logits/chosen": -2.814621925354004, + "logits/rejected": -2.7681021690368652, + "logps/chosen": -254.0991973876953, + "logps/rejected": -227.88961791992188, + "loss": 0.6404, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.3078501522541046, + "rewards/margins": 0.14654883742332458, + "rewards/rejected": -0.4543989598751068, + "step": 305 + }, + { + "epoch": 0.248, + "grad_norm": 2.648085832595825, + "learning_rate": 4.673737323763048e-06, + "logits/chosen": -2.870044708251953, + "logits/rejected": -2.8921992778778076, + "logps/chosen": -322.07037353515625, + "logps/rejected": -309.27874755859375, + "loss": 0.5945, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.23309031128883362, + "rewards/margins": 0.25641053915023804, + "rewards/rejected": -0.48950082063674927, + "step": 310 + }, + { + "epoch": 0.252, + "grad_norm": 2.9251554012298584, + "learning_rate": 4.656284173018144e-06, + "logits/chosen": -2.796626567840576, + "logits/rejected": -2.777113437652588, + "logps/chosen": -303.8788146972656, + "logps/rejected": -336.8778991699219, + "loss": 0.6197, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.22741813957691193, + "rewards/margins": 0.18721428513526917, + "rewards/rejected": -0.4146324098110199, + "step": 315 + }, + { + "epoch": 0.256, + "grad_norm": 3.248079776763916, + "learning_rate": 4.638410650401267e-06, + "logits/chosen": -2.868063449859619, + "logits/rejected": -2.8760440349578857, + "logps/chosen": -310.02423095703125, + "logps/rejected": -324.77301025390625, + "loss": 0.6175, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.21097686886787415, + "rewards/margins": 0.21348261833190918, + "rewards/rejected": -0.42445945739746094, + "step": 320 + }, + { + "epoch": 0.26, + "grad_norm": 3.228531837463379, + "learning_rate": 4.620120240391065e-06, + "logits/chosen": -2.8391566276550293, + "logits/rejected": -2.864546775817871, + "logps/chosen": -333.9083251953125, + "logps/rejected": -310.19256591796875, + "loss": 0.6103, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.18001362681388855, + "rewards/margins": 0.2388877123594284, + "rewards/rejected": -0.41890135407447815, + "step": 325 + }, + { + "epoch": 0.264, + "grad_norm": 4.000072479248047, + "learning_rate": 4.601416508739211e-06, + "logits/chosen": -2.768284320831299, + "logits/rejected": -2.7349295616149902, + "logps/chosen": -294.70703125, + "logps/rejected": -288.8208923339844, + "loss": 0.61, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.18951205909252167, + "rewards/margins": 0.23551206290721893, + "rewards/rejected": -0.4250241816043854, + "step": 330 + }, + { + "epoch": 0.268, + "grad_norm": 4.16896915435791, + "learning_rate": 4.582303101775249e-06, + "logits/chosen": -2.7774980068206787, + "logits/rejected": -2.754678726196289, + "logps/chosen": -302.11602783203125, + "logps/rejected": -277.4075012207031, + "loss": 0.613, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.25999313592910767, + "rewards/margins": 0.24224285781383514, + "rewards/rejected": -0.502236008644104, + "step": 335 + }, + { + "epoch": 0.272, + "grad_norm": 3.032811403274536, + "learning_rate": 4.562783745695738e-06, + "logits/chosen": -2.7642288208007812, + "logits/rejected": -2.8096585273742676, + "logps/chosen": -216.1359100341797, + "logps/rejected": -250.99014282226562, + "loss": 0.6156, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.3114262819290161, + "rewards/margins": 0.21660414338111877, + "rewards/rejected": -0.5280304551124573, + "step": 340 + }, + { + "epoch": 0.276, + "grad_norm": 4.874217987060547, + "learning_rate": 4.542862245837821e-06, + "logits/chosen": -2.868809700012207, + "logits/rejected": -2.817004680633545, + "logps/chosen": -327.43060302734375, + "logps/rejected": -330.1990966796875, + "loss": 0.5829, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.24460521340370178, + "rewards/margins": 0.30477675795555115, + "rewards/rejected": -0.5493819117546082, + "step": 345 + }, + { + "epoch": 0.28, + "grad_norm": 4.413153648376465, + "learning_rate": 4.522542485937369e-06, + "logits/chosen": -2.7310781478881836, + "logits/rejected": -2.7017111778259277, + "logps/chosen": -269.2838134765625, + "logps/rejected": -288.1488952636719, + "loss": 0.6175, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.31993380188941956, + "rewards/margins": 0.23448209464550018, + "rewards/rejected": -0.5544158220291138, + "step": 350 + }, + { + "epoch": 0.284, + "grad_norm": 3.4240481853485107, + "learning_rate": 4.501828427371834e-06, + "logits/chosen": -2.8249359130859375, + "logits/rejected": -2.7783515453338623, + "logps/chosen": -279.47833251953125, + "logps/rejected": -266.2778015136719, + "loss": 0.6243, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2718459963798523, + "rewards/margins": 0.22681903839111328, + "rewards/rejected": -0.4986650049686432, + "step": 355 + }, + { + "epoch": 0.288, + "grad_norm": 4.139309406280518, + "learning_rate": 4.4807241083879774e-06, + "logits/chosen": -2.8446147441864014, + "logits/rejected": -2.8624658584594727, + "logps/chosen": -301.0242614746094, + "logps/rejected": -329.738037109375, + "loss": 0.6215, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.29846441745758057, + "rewards/margins": 0.22665563225746155, + "rewards/rejected": -0.5251200199127197, + "step": 360 + }, + { + "epoch": 0.292, + "grad_norm": 3.9606006145477295, + "learning_rate": 4.4592336433146e-06, + "logits/chosen": -2.829144239425659, + "logits/rejected": -2.8264288902282715, + "logps/chosen": -306.7838439941406, + "logps/rejected": -312.4410095214844, + "loss": 0.6144, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.3578701913356781, + "rewards/margins": 0.22671571373939514, + "rewards/rejected": -0.5845859050750732, + "step": 365 + }, + { + "epoch": 0.296, + "grad_norm": 4.797597885131836, + "learning_rate": 4.437361221760449e-06, + "logits/chosen": -2.8679544925689697, + "logits/rejected": -2.8508830070495605, + "logps/chosen": -314.5845947265625, + "logps/rejected": -294.0763244628906, + "loss": 0.5933, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.30855458974838257, + "rewards/margins": 0.29460665583610535, + "rewards/rejected": -0.6031612157821655, + "step": 370 + }, + { + "epoch": 0.3, + "grad_norm": 4.122452259063721, + "learning_rate": 4.415111107797445e-06, + "logits/chosen": -2.7813663482666016, + "logits/rejected": -2.6964869499206543, + "logps/chosen": -304.09637451171875, + "logps/rejected": -295.1402587890625, + "loss": 0.635, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.27096718549728394, + "rewards/margins": 0.20372644066810608, + "rewards/rejected": -0.4746936857700348, + "step": 375 + }, + { + "epoch": 0.304, + "grad_norm": 3.014270305633545, + "learning_rate": 4.3924876391293915e-06, + "logits/chosen": -2.813854217529297, + "logits/rejected": -2.7791194915771484, + "logps/chosen": -274.0520935058594, + "logps/rejected": -276.63140869140625, + "loss": 0.6145, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.14454008638858795, + "rewards/margins": 0.22425620257854462, + "rewards/rejected": -0.36879628896713257, + "step": 380 + }, + { + "epoch": 0.308, + "grad_norm": 3.01690936088562, + "learning_rate": 4.36949522624633e-06, + "logits/chosen": -2.8495421409606934, + "logits/rejected": -2.8250906467437744, + "logps/chosen": -324.4741516113281, + "logps/rejected": -310.1029968261719, + "loss": 0.5852, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.11460743099451065, + "rewards/margins": 0.2905317544937134, + "rewards/rejected": -0.4051392078399658, + "step": 385 + }, + { + "epoch": 0.312, + "grad_norm": 6.7993245124816895, + "learning_rate": 4.346138351564711e-06, + "logits/chosen": -2.8487377166748047, + "logits/rejected": -2.778311252593994, + "logps/chosen": -363.527587890625, + "logps/rejected": -311.5059509277344, + "loss": 0.6315, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.21368345618247986, + "rewards/margins": 0.20435233414173126, + "rewards/rejected": -0.4180358052253723, + "step": 390 + }, + { + "epoch": 0.316, + "grad_norm": 4.2425737380981445, + "learning_rate": 4.322421568553529e-06, + "logits/chosen": -2.8648476600646973, + "logits/rejected": -2.8160643577575684, + "logps/chosen": -383.7563781738281, + "logps/rejected": -340.1937561035156, + "loss": 0.6138, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1808895766735077, + "rewards/margins": 0.22633692622184753, + "rewards/rejected": -0.40722647309303284, + "step": 395 + }, + { + "epoch": 0.32, + "grad_norm": 3.8491086959838867, + "learning_rate": 4.2983495008466285e-06, + "logits/chosen": -2.879483461380005, + "logits/rejected": -2.8419952392578125, + "logps/chosen": -320.96197509765625, + "logps/rejected": -318.02056884765625, + "loss": 0.5689, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.22649240493774414, + "rewards/margins": 0.3563699424266815, + "rewards/rejected": -0.5828623175621033, + "step": 400 + }, + { + "epoch": 0.32, + "eval_logits/chosen": -2.8437469005584717, + "eval_logits/rejected": -2.8083014488220215, + "eval_logps/chosen": -312.95733642578125, + "eval_logps/rejected": -305.2159423828125, + "eval_loss": 0.6026533246040344, + "eval_rewards/accuracies": 0.6944444179534912, + "eval_rewards/chosen": -0.29719212651252747, + "eval_rewards/margins": 0.27467086911201477, + "eval_rewards/rejected": -0.5718629360198975, + "eval_runtime": 166.7775, + "eval_samples_per_second": 2.998, + "eval_steps_per_second": 0.378, + "step": 400 + }, + { + "epoch": 0.324, + "grad_norm": 5.923827648162842, + "learning_rate": 4.273926841341303e-06, + "logits/chosen": -2.8288912773132324, + "logits/rejected": -2.811527967453003, + "logps/chosen": -270.85467529296875, + "logps/rejected": -300.5176086425781, + "loss": 0.6137, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3265763223171234, + "rewards/margins": 0.2849840521812439, + "rewards/rejected": -0.6115604639053345, + "step": 405 + }, + { + "epoch": 0.328, + "grad_norm": 4.179617404937744, + "learning_rate": 4.249158351283414e-06, + "logits/chosen": -2.830479860305786, + "logits/rejected": -2.7938389778137207, + "logps/chosen": -298.1662902832031, + "logps/rejected": -311.0763244628906, + "loss": 0.6207, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3962728977203369, + "rewards/margins": 0.26946958899497986, + "rewards/rejected": -0.6657425165176392, + "step": 410 + }, + { + "epoch": 0.332, + "grad_norm": 4.05703592300415, + "learning_rate": 4.224048859339175e-06, + "logits/chosen": -2.8106844425201416, + "logits/rejected": -2.793975591659546, + "logps/chosen": -320.8597717285156, + "logps/rejected": -314.65643310546875, + "loss": 0.5808, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.2955939471721649, + "rewards/margins": 0.317967027425766, + "rewards/rejected": -0.6135609745979309, + "step": 415 + }, + { + "epoch": 0.336, + "grad_norm": 3.603731632232666, + "learning_rate": 4.198603260653792e-06, + "logits/chosen": -2.828977108001709, + "logits/rejected": -2.8087880611419678, + "logps/chosen": -318.9561767578125, + "logps/rejected": -296.2648010253906, + "loss": 0.6219, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.312124639749527, + "rewards/margins": 0.2338072955608368, + "rewards/rejected": -0.5459319353103638, + "step": 420 + }, + { + "epoch": 0.34, + "grad_norm": 3.502253293991089, + "learning_rate": 4.172826515897146e-06, + "logits/chosen": -2.840507984161377, + "logits/rejected": -2.8030855655670166, + "logps/chosen": -284.6482238769531, + "logps/rejected": -301.00970458984375, + "loss": 0.5747, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.24788649380207062, + "rewards/margins": 0.3554701507091522, + "rewards/rejected": -0.6033565998077393, + "step": 425 + }, + { + "epoch": 0.344, + "grad_norm": 3.493734359741211, + "learning_rate": 4.146723650296701e-06, + "logits/chosen": -2.842768430709839, + "logits/rejected": -2.8307838439941406, + "logps/chosen": -303.18511962890625, + "logps/rejected": -299.94586181640625, + "loss": 0.5988, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.238745778799057, + "rewards/margins": 0.2844436764717102, + "rewards/rejected": -0.5231894254684448, + "step": 430 + }, + { + "epoch": 0.348, + "grad_norm": 4.337499618530273, + "learning_rate": 4.120299752657828e-06, + "logits/chosen": -2.820674180984497, + "logits/rejected": -2.8131017684936523, + "logps/chosen": -309.2215270996094, + "logps/rejected": -302.0687561035156, + "loss": 0.5918, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.20002242922782898, + "rewards/margins": 0.3058350086212158, + "rewards/rejected": -0.5058574080467224, + "step": 435 + }, + { + "epoch": 0.352, + "grad_norm": 4.533146381378174, + "learning_rate": 4.093559974371725e-06, + "logits/chosen": -2.8260464668273926, + "logits/rejected": -2.8369338512420654, + "logps/chosen": -313.92413330078125, + "logps/rejected": -341.5757141113281, + "loss": 0.5944, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.24169588088989258, + "rewards/margins": 0.32093319296836853, + "rewards/rejected": -0.562628984451294, + "step": 440 + }, + { + "epoch": 0.356, + "grad_norm": 6.319594860076904, + "learning_rate": 4.066509528411151e-06, + "logits/chosen": -2.7408385276794434, + "logits/rejected": -2.70278000831604, + "logps/chosen": -280.9513854980469, + "logps/rejected": -315.61260986328125, + "loss": 0.5481, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.34108084440231323, + "rewards/margins": 0.41155967116355896, + "rewards/rejected": -0.7526406049728394, + "step": 445 + }, + { + "epoch": 0.36, + "grad_norm": 7.433955192565918, + "learning_rate": 4.039153688314146e-06, + "logits/chosen": -2.865485906600952, + "logits/rejected": -2.8098623752593994, + "logps/chosen": -350.09820556640625, + "logps/rejected": -329.5234680175781, + "loss": 0.6008, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.49973931908607483, + "rewards/margins": 0.3033995032310486, + "rewards/rejected": -0.8031389117240906, + "step": 450 + }, + { + "epoch": 0.364, + "grad_norm": 4.991713047027588, + "learning_rate": 4.011497787155938e-06, + "logits/chosen": -2.777055501937866, + "logits/rejected": -2.717963695526123, + "logps/chosen": -340.73223876953125, + "logps/rejected": -334.4681091308594, + "loss": 0.5899, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.6034590005874634, + "rewards/margins": 0.3287494480609894, + "rewards/rejected": -0.9322085380554199, + "step": 455 + }, + { + "epoch": 0.368, + "grad_norm": 4.300924301147461, + "learning_rate": 3.983547216509254e-06, + "logits/chosen": -2.8502197265625, + "logits/rejected": -2.8119924068450928, + "logps/chosen": -382.5280456542969, + "logps/rejected": -336.8903503417969, + "loss": 0.5755, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.48744526505470276, + "rewards/margins": 0.3749392628669739, + "rewards/rejected": -0.8623844981193542, + "step": 460 + }, + { + "epoch": 0.372, + "grad_norm": 4.1107025146484375, + "learning_rate": 3.955307425393224e-06, + "logits/chosen": -2.875734329223633, + "logits/rejected": -2.834575891494751, + "logps/chosen": -360.67852783203125, + "logps/rejected": -366.3440856933594, + "loss": 0.5174, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.29663369059562683, + "rewards/margins": 0.5048123002052307, + "rewards/rejected": -0.8014459609985352, + "step": 465 + }, + { + "epoch": 0.376, + "grad_norm": 5.430475234985352, + "learning_rate": 3.92678391921108e-06, + "logits/chosen": -2.725780963897705, + "logits/rejected": -2.7029075622558594, + "logps/chosen": -364.1918640136719, + "logps/rejected": -378.3152160644531, + "loss": 0.5719, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.42064231634140015, + "rewards/margins": 0.38249263167381287, + "rewards/rejected": -0.8031350374221802, + "step": 470 + }, + { + "epoch": 0.38, + "grad_norm": 6.495006561279297, + "learning_rate": 3.897982258676867e-06, + "logits/chosen": -2.775744915008545, + "logits/rejected": -2.769195318222046, + "logps/chosen": -308.8157043457031, + "logps/rejected": -341.82037353515625, + "loss": 0.5823, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.36774376034736633, + "rewards/margins": 0.3316097855567932, + "rewards/rejected": -0.6993535161018372, + "step": 475 + }, + { + "epoch": 0.384, + "grad_norm": 7.480860233306885, + "learning_rate": 3.868908058731376e-06, + "logits/chosen": -2.7915594577789307, + "logits/rejected": -2.742253065109253, + "logps/chosen": -352.02996826171875, + "logps/rejected": -327.95928955078125, + "loss": 0.6819, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -0.5273100733757019, + "rewards/margins": 0.15378466248512268, + "rewards/rejected": -0.681094765663147, + "step": 480 + }, + { + "epoch": 0.388, + "grad_norm": 8.06702709197998, + "learning_rate": 3.839566987447492e-06, + "logits/chosen": -2.7685608863830566, + "logits/rejected": -2.7417702674865723, + "logps/chosen": -345.76495361328125, + "logps/rejected": -346.05426025390625, + "loss": 0.573, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5824685096740723, + "rewards/margins": 0.387678325176239, + "rewards/rejected": -0.9701469540596008, + "step": 485 + }, + { + "epoch": 0.392, + "grad_norm": 5.584991931915283, + "learning_rate": 3.8099647649251984e-06, + "logits/chosen": -2.816774845123291, + "logits/rejected": -2.766758441925049, + "logps/chosen": -344.7107849121094, + "logps/rejected": -350.0550842285156, + "loss": 0.621, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.6366808414459229, + "rewards/margins": 0.28556036949157715, + "rewards/rejected": -0.9222410917282104, + "step": 490 + }, + { + "epoch": 0.396, + "grad_norm": 8.032232284545898, + "learning_rate": 3.780107162176429e-06, + "logits/chosen": -2.7901835441589355, + "logits/rejected": -2.771435260772705, + "logps/chosen": -359.55804443359375, + "logps/rejected": -320.9138488769531, + "loss": 0.5898, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.7569794058799744, + "rewards/margins": 0.3364938199520111, + "rewards/rejected": -1.0934733152389526, + "step": 495 + }, + { + "epoch": 0.4, + "grad_norm": 6.636690616607666, + "learning_rate": 3.7500000000000005e-06, + "logits/chosen": -2.7686400413513184, + "logits/rejected": -2.747738838195801, + "logps/chosen": -380.00469970703125, + "logps/rejected": -391.73797607421875, + "loss": 0.5689, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6767264008522034, + "rewards/margins": 0.42483949661254883, + "rewards/rejected": -1.1015657186508179, + "step": 500 + }, + { + "epoch": 0.4, + "eval_logits/chosen": -2.7560079097747803, + "eval_logits/rejected": -2.7151803970336914, + "eval_logps/chosen": -349.3812255859375, + "eval_logps/rejected": -355.066162109375, + "eval_loss": 0.5749732851982117, + "eval_rewards/accuracies": 0.7242063283920288, + "eval_rewards/chosen": -0.6614311933517456, + "eval_rewards/margins": 0.40893420577049255, + "eval_rewards/rejected": -1.0703654289245605, + "eval_runtime": 166.5841, + "eval_samples_per_second": 3.001, + "eval_steps_per_second": 0.378, + "step": 500 + }, + { + "epoch": 0.404, + "grad_norm": 5.858213901519775, + "learning_rate": 3.7196491478468322e-06, + "logits/chosen": -2.6860787868499756, + "logits/rejected": -2.702346086502075, + "logps/chosen": -351.7030944824219, + "logps/rejected": -389.79852294921875, + "loss": 0.5747, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.7111638784408569, + "rewards/margins": 0.4305337369441986, + "rewards/rejected": -1.141697645187378, + "step": 505 + }, + { + "epoch": 0.408, + "grad_norm": 6.421969890594482, + "learning_rate": 3.689060522675689e-06, + "logits/chosen": -2.7720227241516113, + "logits/rejected": -2.7550883293151855, + "logps/chosen": -335.5453186035156, + "logps/rejected": -353.2554931640625, + "loss": 0.6084, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.47883662581443787, + "rewards/margins": 0.3589397370815277, + "rewards/rejected": -0.8377763628959656, + "step": 510 + }, + { + "epoch": 0.412, + "grad_norm": 6.96730375289917, + "learning_rate": 3.658240087799655e-06, + "logits/chosen": -2.702028751373291, + "logits/rejected": -2.715606689453125, + "logps/chosen": -309.8144836425781, + "logps/rejected": -362.7301330566406, + "loss": 0.5446, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.4930266737937927, + "rewards/margins": 0.47733306884765625, + "rewards/rejected": -0.9703596830368042, + "step": 515 + }, + { + "epoch": 0.416, + "grad_norm": 4.9304680824279785, + "learning_rate": 3.627193851723577e-06, + "logits/chosen": -2.7465412616729736, + "logits/rejected": -2.7204620838165283, + "logps/chosen": -352.72674560546875, + "logps/rejected": -372.55242919921875, + "loss": 0.6349, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.7522457242012024, + "rewards/margins": 0.3193085789680481, + "rewards/rejected": -1.0715543031692505, + "step": 520 + }, + { + "epoch": 0.42, + "grad_norm": 6.871084213256836, + "learning_rate": 3.595927866972694e-06, + "logits/chosen": -2.708409309387207, + "logits/rejected": -2.7058815956115723, + "logps/chosen": -292.47271728515625, + "logps/rejected": -334.9390869140625, + "loss": 0.5762, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.6716808080673218, + "rewards/margins": 0.46342235803604126, + "rewards/rejected": -1.1351032257080078, + "step": 525 + }, + { + "epoch": 0.424, + "grad_norm": 6.238135814666748, + "learning_rate": 3.564448228912682e-06, + "logits/chosen": -2.6425788402557373, + "logits/rejected": -2.6337852478027344, + "logps/chosen": -382.01910400390625, + "logps/rejected": -381.7391662597656, + "loss": 0.6097, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.7499979734420776, + "rewards/margins": 0.37163788080215454, + "rewards/rejected": -1.1216356754302979, + "step": 530 + }, + { + "epoch": 0.428, + "grad_norm": 14.889227867126465, + "learning_rate": 3.532761074561355e-06, + "logits/chosen": -2.6252596378326416, + "logits/rejected": -2.573930501937866, + "logps/chosen": -389.1572265625, + "logps/rejected": -428.70489501953125, + "loss": 0.5708, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.7006493806838989, + "rewards/margins": 0.508830189704895, + "rewards/rejected": -1.209479570388794, + "step": 535 + }, + { + "epoch": 0.432, + "grad_norm": 5.182321548461914, + "learning_rate": 3.5008725813922383e-06, + "logits/chosen": -2.7459521293640137, + "logits/rejected": -2.672208309173584, + "logps/chosen": -356.2322998046875, + "logps/rejected": -392.16021728515625, + "loss": 0.5393, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.6245570182800293, + "rewards/margins": 0.5389561057090759, + "rewards/rejected": -1.1635130643844604, + "step": 540 + }, + { + "epoch": 0.436, + "grad_norm": 8.57190990447998, + "learning_rate": 3.4687889661302577e-06, + "logits/chosen": -2.6495566368103027, + "logits/rejected": -2.649183511734009, + "logps/chosen": -315.9193115234375, + "logps/rejected": -356.46417236328125, + "loss": 0.5273, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.6088639497756958, + "rewards/margins": 0.5826314687728882, + "rewards/rejected": -1.1914955377578735, + "step": 545 + }, + { + "epoch": 0.44, + "grad_norm": 6.0855207443237305, + "learning_rate": 3.436516483539781e-06, + "logits/chosen": -2.6810402870178223, + "logits/rejected": -2.6679680347442627, + "logps/chosen": -341.8032531738281, + "logps/rejected": -360.6465759277344, + "loss": 0.6281, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7102030515670776, + "rewards/margins": 0.3800693154335022, + "rewards/rejected": -1.090272307395935, + "step": 550 + }, + { + "epoch": 0.444, + "grad_norm": 6.288063049316406, + "learning_rate": 3.4040614252052305e-06, + "logits/chosen": -2.672840118408203, + "logits/rejected": -2.665916919708252, + "logps/chosen": -357.5086975097656, + "logps/rejected": -378.4448547363281, + "loss": 0.6071, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7066811919212341, + "rewards/margins": 0.38481634855270386, + "rewards/rejected": -1.091497540473938, + "step": 555 + }, + { + "epoch": 0.448, + "grad_norm": 6.352199554443359, + "learning_rate": 3.3714301183045382e-06, + "logits/chosen": -2.631598711013794, + "logits/rejected": -2.574202299118042, + "logps/chosen": -298.14923095703125, + "logps/rejected": -323.47528076171875, + "loss": 0.61, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.6164783239364624, + "rewards/margins": 0.34816139936447144, + "rewards/rejected": -0.9646397829055786, + "step": 560 + }, + { + "epoch": 0.452, + "grad_norm": 4.746293067932129, + "learning_rate": 3.338628924375638e-06, + "logits/chosen": -2.759838581085205, + "logits/rejected": -2.7202701568603516, + "logps/chosen": -286.07159423828125, + "logps/rejected": -346.5191955566406, + "loss": 0.5194, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.4790007174015045, + "rewards/margins": 0.5368759036064148, + "rewards/rejected": -1.0158765316009521, + "step": 565 + }, + { + "epoch": 0.456, + "grad_norm": 4.951190948486328, + "learning_rate": 3.3056642380762783e-06, + "logits/chosen": -2.7108511924743652, + "logits/rejected": -2.6962389945983887, + "logps/chosen": -269.0116271972656, + "logps/rejected": -301.1402893066406, + "loss": 0.5847, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -0.3556315302848816, + "rewards/margins": 0.43046459555625916, + "rewards/rejected": -0.7860961556434631, + "step": 570 + }, + { + "epoch": 0.46, + "grad_norm": 6.543835639953613, + "learning_rate": 3.272542485937369e-06, + "logits/chosen": -2.6798412799835205, + "logits/rejected": -2.6179356575012207, + "logps/chosen": -285.6194152832031, + "logps/rejected": -290.96771240234375, + "loss": 0.5738, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.43146055936813354, + "rewards/margins": 0.3925173878669739, + "rewards/rejected": -0.8239779472351074, + "step": 575 + }, + { + "epoch": 0.464, + "grad_norm": 4.029048442840576, + "learning_rate": 3.2392701251101172e-06, + "logits/chosen": -2.7277991771698, + "logits/rejected": -2.6769065856933594, + "logps/chosen": -338.96826171875, + "logps/rejected": -356.6761169433594, + "loss": 0.517, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.38630399107933044, + "rewards/margins": 0.5923217535018921, + "rewards/rejected": -0.9786256551742554, + "step": 580 + }, + { + "epoch": 0.468, + "grad_norm": 6.441251277923584, + "learning_rate": 3.205853642107192e-06, + "logits/chosen": -2.6596102714538574, + "logits/rejected": -2.6377530097961426, + "logps/chosen": -295.38067626953125, + "logps/rejected": -321.3651428222656, + "loss": 0.6042, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.518380343914032, + "rewards/margins": 0.3751484155654907, + "rewards/rejected": -0.8935287594795227, + "step": 585 + }, + { + "epoch": 0.472, + "grad_norm": 8.161737442016602, + "learning_rate": 3.1722995515381644e-06, + "logits/chosen": -2.6348493099212646, + "logits/rejected": -2.622722625732422, + "logps/chosen": -333.249755859375, + "logps/rejected": -362.6484069824219, + "loss": 0.5217, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.5237576365470886, + "rewards/margins": 0.5885855555534363, + "rewards/rejected": -1.1123430728912354, + "step": 590 + }, + { + "epoch": 0.476, + "grad_norm": 5.944140911102295, + "learning_rate": 3.1386143948394764e-06, + "logits/chosen": -2.648714780807495, + "logits/rejected": -2.6416144371032715, + "logps/chosen": -329.64849853515625, + "logps/rejected": -402.10260009765625, + "loss": 0.5543, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.7717787027359009, + "rewards/margins": 0.5048314332962036, + "rewards/rejected": -1.2766101360321045, + "step": 595 + }, + { + "epoch": 0.48, + "grad_norm": 7.663615703582764, + "learning_rate": 3.1048047389991693e-06, + "logits/chosen": -2.652611494064331, + "logits/rejected": -2.5794761180877686, + "logps/chosen": -387.5853576660156, + "logps/rejected": -337.8640441894531, + "loss": 0.5884, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6710079908370972, + "rewards/margins": 0.47154727578163147, + "rewards/rejected": -1.1425553560256958, + "step": 600 + }, + { + "epoch": 0.48, + "eval_logits/chosen": -2.6723644733428955, + "eval_logits/rejected": -2.6321890354156494, + "eval_logps/chosen": -352.8876647949219, + "eval_logps/rejected": -375.1052551269531, + "eval_loss": 0.547924816608429, + "eval_rewards/accuracies": 0.7123016119003296, + "eval_rewards/chosen": -0.6964960694313049, + "eval_rewards/margins": 0.5742600560188293, + "eval_rewards/rejected": -1.2707562446594238, + "eval_runtime": 166.6085, + "eval_samples_per_second": 3.001, + "eval_steps_per_second": 0.378, + "step": 600 + }, + { + "epoch": 0.484, + "grad_norm": 7.883712291717529, + "learning_rate": 3.0708771752766397e-06, + "logits/chosen": -2.699237585067749, + "logits/rejected": -2.656472682952881, + "logps/chosen": -396.58935546875, + "logps/rejected": -403.0001525878906, + "loss": 0.5645, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.6690188050270081, + "rewards/margins": 0.4926017224788666, + "rewards/rejected": -1.1616204977035522, + "step": 605 + }, + { + "epoch": 0.488, + "grad_norm": 7.566442489624023, + "learning_rate": 3.0368383179176584e-06, + "logits/chosen": -2.6338205337524414, + "logits/rejected": -2.5693295001983643, + "logps/chosen": -367.86016845703125, + "logps/rejected": -435.5393981933594, + "loss": 0.513, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.7284402847290039, + "rewards/margins": 0.6670497059822083, + "rewards/rejected": -1.395490050315857, + "step": 610 + }, + { + "epoch": 0.492, + "grad_norm": 8.526657104492188, + "learning_rate": 3.002694802864912e-06, + "logits/chosen": -2.618443727493286, + "logits/rejected": -2.598315715789795, + "logps/chosen": -354.45587158203125, + "logps/rejected": -394.5972900390625, + "loss": 0.5754, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.6152892112731934, + "rewards/margins": 0.5398699045181274, + "rewards/rejected": -1.1551592350006104, + "step": 615 + }, + { + "epoch": 0.496, + "grad_norm": 6.0026044845581055, + "learning_rate": 2.9684532864643123e-06, + "logits/chosen": -2.611154317855835, + "logits/rejected": -2.594029188156128, + "logps/chosen": -322.4052734375, + "logps/rejected": -360.61016845703125, + "loss": 0.5007, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.49695730209350586, + "rewards/margins": 0.6773632764816284, + "rewards/rejected": -1.1743205785751343, + "step": 620 + }, + { + "epoch": 0.5, + "grad_norm": 19.88625144958496, + "learning_rate": 2.9341204441673267e-06, + "logits/chosen": -2.6633830070495605, + "logits/rejected": -2.670114040374756, + "logps/chosen": -366.9710388183594, + "logps/rejected": -338.7015075683594, + "loss": 0.667, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.5960913300514221, + "rewards/margins": 0.2840344309806824, + "rewards/rejected": -0.8801258206367493, + "step": 625 + }, + { + "epoch": 0.504, + "grad_norm": 14.67087173461914, + "learning_rate": 2.8997029692295875e-06, + "logits/chosen": -2.6533138751983643, + "logits/rejected": -2.623382329940796, + "logps/chosen": -282.66400146484375, + "logps/rejected": -314.5120849609375, + "loss": 0.6115, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.46091246604919434, + "rewards/margins": 0.3925257921218872, + "rewards/rejected": -0.8534382581710815, + "step": 630 + }, + { + "epoch": 0.508, + "grad_norm": 5.918943405151367, + "learning_rate": 2.8652075714060296e-06, + "logits/chosen": -2.6715197563171387, + "logits/rejected": -2.688370704650879, + "logps/chosen": -284.8374938964844, + "logps/rejected": -335.3628845214844, + "loss": 0.5706, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3965570032596588, + "rewards/margins": 0.44703879952430725, + "rewards/rejected": -0.8435958027839661, + "step": 635 + }, + { + "epoch": 0.512, + "grad_norm": 8.843329429626465, + "learning_rate": 2.8306409756428067e-06, + "logits/chosen": -2.623734951019287, + "logits/rejected": -2.5950891971588135, + "logps/chosen": -270.8127746582031, + "logps/rejected": -276.7856750488281, + "loss": 0.5817, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.30773457884788513, + "rewards/margins": 0.39293187856674194, + "rewards/rejected": -0.7006665468215942, + "step": 640 + }, + { + "epoch": 0.516, + "grad_norm": 7.708199501037598, + "learning_rate": 2.7960099207662535e-06, + "logits/chosen": -2.634336471557617, + "logits/rejected": -2.602273941040039, + "logps/chosen": -293.1026611328125, + "logps/rejected": -321.48004150390625, + "loss": 0.5566, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.4055629372596741, + "rewards/margins": 0.4888216555118561, + "rewards/rejected": -0.8943845629692078, + "step": 645 + }, + { + "epoch": 0.52, + "grad_norm": 5.657192230224609, + "learning_rate": 2.761321158169134e-06, + "logits/chosen": -2.6778979301452637, + "logits/rejected": -2.681678533554077, + "logps/chosen": -330.0281066894531, + "logps/rejected": -331.8005065917969, + "loss": 0.5934, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.48604241013526917, + "rewards/margins": 0.386027991771698, + "rewards/rejected": -0.8720704317092896, + "step": 650 + }, + { + "epoch": 0.524, + "grad_norm": 6.853002071380615, + "learning_rate": 2.726581450494451e-06, + "logits/chosen": -2.6293766498565674, + "logits/rejected": -2.621476173400879, + "logps/chosen": -326.91363525390625, + "logps/rejected": -339.2771911621094, + "loss": 0.529, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.31295642256736755, + "rewards/margins": 0.5676072835922241, + "rewards/rejected": -0.8805637359619141, + "step": 655 + }, + { + "epoch": 0.528, + "grad_norm": 5.255690574645996, + "learning_rate": 2.6917975703170466e-06, + "logits/chosen": -2.6737003326416016, + "logits/rejected": -2.660823106765747, + "logps/chosen": -332.20001220703125, + "logps/rejected": -395.01177978515625, + "loss": 0.4708, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.47576016187667847, + "rewards/margins": 0.7829462885856628, + "rewards/rejected": -1.2587064504623413, + "step": 660 + }, + { + "epoch": 0.532, + "grad_norm": 6.153021335601807, + "learning_rate": 2.6569762988232838e-06, + "logits/chosen": -2.596040725708008, + "logits/rejected": -2.6018855571746826, + "logps/chosen": -314.1906433105469, + "logps/rejected": -368.1581115722656, + "loss": 0.5719, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.6212738752365112, + "rewards/margins": 0.5093709826469421, + "rewards/rejected": -1.1306449174880981, + "step": 665 + }, + { + "epoch": 0.536, + "grad_norm": 12.637703895568848, + "learning_rate": 2.6221244244890336e-06, + "logits/chosen": -2.648380756378174, + "logits/rejected": -2.5612568855285645, + "logps/chosen": -381.6895751953125, + "logps/rejected": -417.946044921875, + "loss": 0.5273, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.9110687375068665, + "rewards/margins": 0.6113173961639404, + "rewards/rejected": -1.5223863124847412, + "step": 670 + }, + { + "epoch": 0.54, + "grad_norm": 15.107666015625, + "learning_rate": 2.587248741756253e-06, + "logits/chosen": -2.6709227561950684, + "logits/rejected": -2.6540889739990234, + "logps/chosen": -357.17413330078125, + "logps/rejected": -416.6585998535156, + "loss": 0.5336, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.7724355459213257, + "rewards/margins": 0.6448327898979187, + "rewards/rejected": -1.41726815700531, + "step": 675 + }, + { + "epoch": 0.544, + "grad_norm": 5.885760307312012, + "learning_rate": 2.5523560497083927e-06, + "logits/chosen": -2.681267023086548, + "logits/rejected": -2.6446001529693604, + "logps/chosen": -372.6912536621094, + "logps/rejected": -416.1136779785156, + "loss": 0.5489, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.9625101089477539, + "rewards/margins": 0.5871840715408325, + "rewards/rejected": -1.549694299697876, + "step": 680 + }, + { + "epoch": 0.548, + "grad_norm": 9.579086303710938, + "learning_rate": 2.517453150744904e-06, + "logits/chosen": -2.6798925399780273, + "logits/rejected": -2.6284408569335938, + "logps/chosen": -407.7876892089844, + "logps/rejected": -411.8699645996094, + "loss": 0.6101, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.9496753811836243, + "rewards/margins": 0.48396366834640503, + "rewards/rejected": -1.4336390495300293, + "step": 685 + }, + { + "epoch": 0.552, + "grad_norm": 6.947661399841309, + "learning_rate": 2.482546849255096e-06, + "logits/chosen": -2.6654515266418457, + "logits/rejected": -2.622973918914795, + "logps/chosen": -384.18719482421875, + "logps/rejected": -447.71356201171875, + "loss": 0.5146, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.9958160519599915, + "rewards/margins": 0.7850648760795593, + "rewards/rejected": -1.7808809280395508, + "step": 690 + }, + { + "epoch": 0.556, + "grad_norm": 5.574069023132324, + "learning_rate": 2.447643950291608e-06, + "logits/chosen": -2.5033955574035645, + "logits/rejected": -2.449953556060791, + "logps/chosen": -345.85723876953125, + "logps/rejected": -342.82061767578125, + "loss": 0.5562, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.8573344349861145, + "rewards/margins": 0.5808383226394653, + "rewards/rejected": -1.438172698020935, + "step": 695 + }, + { + "epoch": 0.56, + "grad_norm": 5.758213520050049, + "learning_rate": 2.4127512582437486e-06, + "logits/chosen": -2.626072406768799, + "logits/rejected": -2.6115658283233643, + "logps/chosen": -373.35693359375, + "logps/rejected": -407.4380187988281, + "loss": 0.5366, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.8556938171386719, + "rewards/margins": 0.6256057620048523, + "rewards/rejected": -1.4812995195388794, + "step": 700 + }, + { + "epoch": 0.56, + "eval_logits/chosen": -2.6541435718536377, + "eval_logits/rejected": -2.6143555641174316, + "eval_logps/chosen": -355.7809143066406, + "eval_logps/rejected": -381.54388427734375, + "eval_loss": 0.5462217330932617, + "eval_rewards/accuracies": 0.7123016119003296, + "eval_rewards/chosen": -0.7254281044006348, + "eval_rewards/margins": 0.6097148060798645, + "eval_rewards/rejected": -1.335142970085144, + "eval_runtime": 166.5614, + "eval_samples_per_second": 3.002, + "eval_steps_per_second": 0.378, + "step": 700 + }, + { + "epoch": 0.564, + "grad_norm": 13.666606903076172, + "learning_rate": 2.377875575510967e-06, + "logits/chosen": -2.5781383514404297, + "logits/rejected": -2.516011953353882, + "logps/chosen": -360.7132873535156, + "logps/rejected": -367.84564208984375, + "loss": 0.587, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.8543895483016968, + "rewards/margins": 0.5572708249092102, + "rewards/rejected": -1.4116604328155518, + "step": 705 + }, + { + "epoch": 0.568, + "grad_norm": 15.681051254272461, + "learning_rate": 2.3430237011767166e-06, + "logits/chosen": -2.6835620403289795, + "logits/rejected": -2.6478209495544434, + "logps/chosen": -351.29498291015625, + "logps/rejected": -375.1694030761719, + "loss": 0.579, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.7802821397781372, + "rewards/margins": 0.47849854826927185, + "rewards/rejected": -1.2587807178497314, + "step": 710 + }, + { + "epoch": 0.572, + "grad_norm": 8.994268417358398, + "learning_rate": 2.3082024296829538e-06, + "logits/chosen": -2.6145269870758057, + "logits/rejected": -2.5766470432281494, + "logps/chosen": -295.27313232421875, + "logps/rejected": -390.36279296875, + "loss": 0.4613, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.6295270919799805, + "rewards/margins": 0.8818346858024597, + "rewards/rejected": -1.511361837387085, + "step": 715 + }, + { + "epoch": 0.576, + "grad_norm": 7.825490474700928, + "learning_rate": 2.2734185495055503e-06, + "logits/chosen": -2.672497272491455, + "logits/rejected": -2.5945022106170654, + "logps/chosen": -352.99005126953125, + "logps/rejected": -351.3237609863281, + "loss": 0.5693, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.6252859830856323, + "rewards/margins": 0.4798332750797272, + "rewards/rejected": -1.1051193475723267, + "step": 720 + }, + { + "epoch": 0.58, + "grad_norm": 7.579111099243164, + "learning_rate": 2.238678841830867e-06, + "logits/chosen": -2.6132864952087402, + "logits/rejected": -2.5825414657592773, + "logps/chosen": -348.1910705566406, + "logps/rejected": -377.7493896484375, + "loss": 0.5637, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.5163500905036926, + "rewards/margins": 0.5300666689872742, + "rewards/rejected": -1.0464167594909668, + "step": 725 + }, + { + "epoch": 0.584, + "grad_norm": 8.520023345947266, + "learning_rate": 2.2039900792337477e-06, + "logits/chosen": -2.6524438858032227, + "logits/rejected": -2.6290249824523926, + "logps/chosen": -356.4406433105469, + "logps/rejected": -374.92156982421875, + "loss": 0.5875, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.6103891730308533, + "rewards/margins": 0.5113776922225952, + "rewards/rejected": -1.1217668056488037, + "step": 730 + }, + { + "epoch": 0.588, + "grad_norm": 4.65096378326416, + "learning_rate": 2.1693590243571937e-06, + "logits/chosen": -2.6716837882995605, + "logits/rejected": -2.613300323486328, + "logps/chosen": -326.7117614746094, + "logps/rejected": -364.78997802734375, + "loss": 0.5516, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6327379941940308, + "rewards/margins": 0.6464030742645264, + "rewards/rejected": -1.2791410684585571, + "step": 735 + }, + { + "epoch": 0.592, + "grad_norm": 8.322911262512207, + "learning_rate": 2.134792428593971e-06, + "logits/chosen": -2.5958149433135986, + "logits/rejected": -2.5660290718078613, + "logps/chosen": -307.79656982421875, + "logps/rejected": -354.429443359375, + "loss": 0.5574, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.7369760870933533, + "rewards/margins": 0.45006656646728516, + "rewards/rejected": -1.1870427131652832, + "step": 740 + }, + { + "epoch": 0.596, + "grad_norm": 7.646263599395752, + "learning_rate": 2.1002970307704134e-06, + "logits/chosen": -2.7141809463500977, + "logits/rejected": -2.65040922164917, + "logps/chosen": -403.737548828125, + "logps/rejected": -440.7948303222656, + "loss": 0.5606, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.694787859916687, + "rewards/margins": 0.6611425280570984, + "rewards/rejected": -1.3559304475784302, + "step": 745 + }, + { + "epoch": 0.6, + "grad_norm": 6.563836097717285, + "learning_rate": 2.0658795558326745e-06, + "logits/chosen": -2.6275224685668945, + "logits/rejected": -2.6394450664520264, + "logps/chosen": -353.59765625, + "logps/rejected": -409.802001953125, + "loss": 0.517, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.674094021320343, + "rewards/margins": 0.7538131475448608, + "rewards/rejected": -1.427907109260559, + "step": 750 + }, + { + "epoch": 0.604, + "grad_norm": 5.990789890289307, + "learning_rate": 2.031546713535688e-06, + "logits/chosen": -2.631734848022461, + "logits/rejected": -2.5736083984375, + "logps/chosen": -346.9231872558594, + "logps/rejected": -393.51727294921875, + "loss": 0.535, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6008895039558411, + "rewards/margins": 0.6397835612297058, + "rewards/rejected": -1.240673303604126, + "step": 755 + }, + { + "epoch": 0.608, + "grad_norm": 10.244848251342773, + "learning_rate": 1.997305197135089e-06, + "logits/chosen": -2.5579869747161865, + "logits/rejected": -2.5684762001037598, + "logps/chosen": -278.13470458984375, + "logps/rejected": -336.5936584472656, + "loss": 0.5631, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6302391886711121, + "rewards/margins": 0.5114450454711914, + "rewards/rejected": -1.1416842937469482, + "step": 760 + }, + { + "epoch": 0.612, + "grad_norm": 7.769974708557129, + "learning_rate": 1.963161682082342e-06, + "logits/chosen": -2.536005735397339, + "logits/rejected": -2.584683656692505, + "logps/chosen": -334.3309020996094, + "logps/rejected": -362.7339782714844, + "loss": 0.548, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.5386423468589783, + "rewards/margins": 0.5726658701896667, + "rewards/rejected": -1.111308217048645, + "step": 765 + }, + { + "epoch": 0.616, + "grad_norm": 4.755601406097412, + "learning_rate": 1.9291228247233607e-06, + "logits/chosen": -2.5625104904174805, + "logits/rejected": -2.5171782970428467, + "logps/chosen": -320.0146484375, + "logps/rejected": -359.59197998046875, + "loss": 0.543, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.3926158845424652, + "rewards/margins": 0.5113145112991333, + "rewards/rejected": -0.9039304852485657, + "step": 770 + }, + { + "epoch": 0.62, + "grad_norm": 8.86306095123291, + "learning_rate": 1.895195261000831e-06, + "logits/chosen": -2.6329500675201416, + "logits/rejected": -2.58791446685791, + "logps/chosen": -342.7969665527344, + "logps/rejected": -399.5611267089844, + "loss": 0.5278, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.41523486375808716, + "rewards/margins": 0.5993978977203369, + "rewards/rejected": -1.0146328210830688, + "step": 775 + }, + { + "epoch": 0.624, + "grad_norm": 7.572498321533203, + "learning_rate": 1.8613856051605242e-06, + "logits/chosen": -2.4784908294677734, + "logits/rejected": -2.511237621307373, + "logps/chosen": -283.90972900390625, + "logps/rejected": -327.43121337890625, + "loss": 0.5407, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.45636481046676636, + "rewards/margins": 0.4989432394504547, + "rewards/rejected": -0.9553079605102539, + "step": 780 + }, + { + "epoch": 0.628, + "grad_norm": 4.849424362182617, + "learning_rate": 1.827700448461836e-06, + "logits/chosen": -2.6728811264038086, + "logits/rejected": -2.5995230674743652, + "logps/chosen": -365.33587646484375, + "logps/rejected": -384.79388427734375, + "loss": 0.5746, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.6461849212646484, + "rewards/margins": 0.504442572593689, + "rewards/rejected": -1.150627613067627, + "step": 785 + }, + { + "epoch": 0.632, + "grad_norm": 4.748244285583496, + "learning_rate": 1.7941463578928088e-06, + "logits/chosen": -2.594512939453125, + "logits/rejected": -2.5624542236328125, + "logps/chosen": -407.58221435546875, + "logps/rejected": -416.52752685546875, + "loss": 0.5731, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.626873791217804, + "rewards/margins": 0.539578914642334, + "rewards/rejected": -1.1664526462554932, + "step": 790 + }, + { + "epoch": 0.636, + "grad_norm": 6.187716007232666, + "learning_rate": 1.7607298748898844e-06, + "logits/chosen": -2.6329944133758545, + "logits/rejected": -2.623441219329834, + "logps/chosen": -341.26910400390625, + "logps/rejected": -384.6199645996094, + "loss": 0.5801, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.59977126121521, + "rewards/margins": 0.5203127264976501, + "rewards/rejected": -1.1200840473175049, + "step": 795 + }, + { + "epoch": 0.64, + "grad_norm": 7.730658054351807, + "learning_rate": 1.7274575140626318e-06, + "logits/chosen": -2.5077157020568848, + "logits/rejected": -2.4561638832092285, + "logps/chosen": -329.1211853027344, + "logps/rejected": -394.10760498046875, + "loss": 0.542, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.562198281288147, + "rewards/margins": 0.5235614776611328, + "rewards/rejected": -1.0857596397399902, + "step": 800 + }, + { + "epoch": 0.64, + "eval_logits/chosen": -2.616276264190674, + "eval_logits/rejected": -2.5757086277008057, + "eval_logps/chosen": -352.43634033203125, + "eval_logps/rejected": -374.8915100097656, + "eval_loss": 0.5450887680053711, + "eval_rewards/accuracies": 0.726190447807312, + "eval_rewards/chosen": -0.6919824481010437, + "eval_rewards/margins": 0.5766366124153137, + "eval_rewards/rejected": -1.2686189413070679, + "eval_runtime": 166.5658, + "eval_samples_per_second": 3.002, + "eval_steps_per_second": 0.378, + "step": 800 + }, + { + "epoch": 0.644, + "grad_norm": 5.391225337982178, + "learning_rate": 1.6943357619237227e-06, + "logits/chosen": -2.571699619293213, + "logits/rejected": -2.5610146522521973, + "logps/chosen": -340.8096618652344, + "logps/rejected": -366.74652099609375, + "loss": 0.4952, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.6746693849563599, + "rewards/margins": 0.645226240158081, + "rewards/rejected": -1.319895625114441, + "step": 805 + }, + { + "epoch": 0.648, + "grad_norm": 7.21277379989624, + "learning_rate": 1.661371075624363e-06, + "logits/chosen": -2.5949785709381104, + "logits/rejected": -2.6454100608825684, + "logps/chosen": -344.26763916015625, + "logps/rejected": -466.04693603515625, + "loss": 0.5694, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9223111867904663, + "rewards/margins": 0.545967161655426, + "rewards/rejected": -1.468278169631958, + "step": 810 + }, + { + "epoch": 0.652, + "grad_norm": 10.09349536895752, + "learning_rate": 1.6285698816954626e-06, + "logits/chosen": -2.614025115966797, + "logits/rejected": -2.5793397426605225, + "logps/chosen": -371.01141357421875, + "logps/rejected": -389.6199035644531, + "loss": 0.5079, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.7361218333244324, + "rewards/margins": 0.6544822454452515, + "rewards/rejected": -1.3906042575836182, + "step": 815 + }, + { + "epoch": 0.656, + "grad_norm": 6.835726737976074, + "learning_rate": 1.5959385747947697e-06, + "logits/chosen": -2.546776294708252, + "logits/rejected": -2.489529848098755, + "logps/chosen": -324.41766357421875, + "logps/rejected": -338.0316162109375, + "loss": 0.5675, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.912179172039032, + "rewards/margins": 0.5451852083206177, + "rewards/rejected": -1.4573644399642944, + "step": 820 + }, + { + "epoch": 0.66, + "grad_norm": 9.413851737976074, + "learning_rate": 1.56348351646022e-06, + "logits/chosen": -2.4420504570007324, + "logits/rejected": -2.3995440006256104, + "logps/chosen": -330.77532958984375, + "logps/rejected": -375.870849609375, + "loss": 0.5706, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.9180534482002258, + "rewards/margins": 0.5257332921028137, + "rewards/rejected": -1.443786859512329, + "step": 825 + }, + { + "epoch": 0.664, + "grad_norm": 7.294904708862305, + "learning_rate": 1.5312110338697427e-06, + "logits/chosen": -2.534151792526245, + "logits/rejected": -2.457547426223755, + "logps/chosen": -343.48895263671875, + "logps/rejected": -404.5125427246094, + "loss": 0.5288, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.9970987439155579, + "rewards/margins": 0.7042299509048462, + "rewards/rejected": -1.7013286352157593, + "step": 830 + }, + { + "epoch": 0.668, + "grad_norm": 7.913896083831787, + "learning_rate": 1.4991274186077632e-06, + "logits/chosen": -2.5275869369506836, + "logits/rejected": -2.5172677040100098, + "logps/chosen": -372.0498046875, + "logps/rejected": -428.5511779785156, + "loss": 0.5155, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.8607346415519714, + "rewards/margins": 0.6735731959342957, + "rewards/rejected": -1.5343079566955566, + "step": 835 + }, + { + "epoch": 0.672, + "grad_norm": 9.02523136138916, + "learning_rate": 1.467238925438646e-06, + "logits/chosen": -2.542417287826538, + "logits/rejected": -2.4911322593688965, + "logps/chosen": -409.1072082519531, + "logps/rejected": -441.38818359375, + "loss": 0.6043, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.8746612668037415, + "rewards/margins": 0.6565597653388977, + "rewards/rejected": -1.5312209129333496, + "step": 840 + }, + { + "epoch": 0.676, + "grad_norm": 7.484790325164795, + "learning_rate": 1.4355517710873184e-06, + "logits/chosen": -2.505631446838379, + "logits/rejected": -2.4716596603393555, + "logps/chosen": -369.3199157714844, + "logps/rejected": -385.47271728515625, + "loss": 0.5162, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.8367518186569214, + "rewards/margins": 0.6746172904968262, + "rewards/rejected": -1.5113691091537476, + "step": 845 + }, + { + "epoch": 0.68, + "grad_norm": 14.492733001708984, + "learning_rate": 1.4040721330273063e-06, + "logits/chosen": -2.4774107933044434, + "logits/rejected": -2.4899094104766846, + "logps/chosen": -358.19085693359375, + "logps/rejected": -413.7503967285156, + "loss": 0.6545, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.9458563923835754, + "rewards/margins": 0.5173967480659485, + "rewards/rejected": -1.4632532596588135, + "step": 850 + }, + { + "epoch": 0.684, + "grad_norm": 9.713050842285156, + "learning_rate": 1.3728061482764238e-06, + "logits/chosen": -2.6106739044189453, + "logits/rejected": -2.6073949337005615, + "logps/chosen": -393.0316162109375, + "logps/rejected": -462.81964111328125, + "loss": 0.6226, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7931762933731079, + "rewards/margins": 0.5521876215934753, + "rewards/rejected": -1.3453638553619385, + "step": 855 + }, + { + "epoch": 0.688, + "grad_norm": 8.317811012268066, + "learning_rate": 1.3417599122003464e-06, + "logits/chosen": -2.5949435234069824, + "logits/rejected": -2.5823256969451904, + "logps/chosen": -336.19964599609375, + "logps/rejected": -373.7811279296875, + "loss": 0.6114, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.7611545324325562, + "rewards/margins": 0.44461917877197266, + "rewards/rejected": -1.2057737112045288, + "step": 860 + }, + { + "epoch": 0.692, + "grad_norm": 7.705052852630615, + "learning_rate": 1.3109394773243117e-06, + "logits/chosen": -2.5213277339935303, + "logits/rejected": -2.5227763652801514, + "logps/chosen": -378.220458984375, + "logps/rejected": -433.517822265625, + "loss": 0.5362, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.8036476373672485, + "rewards/margins": 0.7761087417602539, + "rewards/rejected": -1.5797563791275024, + "step": 865 + }, + { + "epoch": 0.696, + "grad_norm": 13.364889144897461, + "learning_rate": 1.280350852153168e-06, + "logits/chosen": -2.5971102714538574, + "logits/rejected": -2.5233542919158936, + "logps/chosen": -357.8860778808594, + "logps/rejected": -374.4933776855469, + "loss": 0.5301, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.7840938568115234, + "rewards/margins": 0.6052092909812927, + "rewards/rejected": -1.3893029689788818, + "step": 870 + }, + { + "epoch": 0.7, + "grad_norm": 9.326637268066406, + "learning_rate": 1.2500000000000007e-06, + "logits/chosen": -2.5254058837890625, + "logits/rejected": -2.5030879974365234, + "logps/chosen": -361.384521484375, + "logps/rejected": -415.76226806640625, + "loss": 0.5189, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.733026385307312, + "rewards/margins": 0.6997817754745483, + "rewards/rejected": -1.4328081607818604, + "step": 875 + }, + { + "epoch": 0.704, + "grad_norm": 10.961359024047852, + "learning_rate": 1.2198928378235717e-06, + "logits/chosen": -2.572036027908325, + "logits/rejected": -2.561758518218994, + "logps/chosen": -297.1030578613281, + "logps/rejected": -387.46685791015625, + "loss": 0.5146, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.5809527635574341, + "rewards/margins": 0.6962177753448486, + "rewards/rejected": -1.2771704196929932, + "step": 880 + }, + { + "epoch": 0.708, + "grad_norm": 6.878293514251709, + "learning_rate": 1.1900352350748026e-06, + "logits/chosen": -2.54471492767334, + "logits/rejected": -2.507836103439331, + "logps/chosen": -375.3175354003906, + "logps/rejected": -410.13555908203125, + "loss": 0.5035, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.7230747938156128, + "rewards/margins": 0.817081093788147, + "rewards/rejected": -1.5401558876037598, + "step": 885 + }, + { + "epoch": 0.712, + "grad_norm": 10.813645362854004, + "learning_rate": 1.160433012552508e-06, + "logits/chosen": -2.4845707416534424, + "logits/rejected": -2.4921040534973145, + "logps/chosen": -331.4786682128906, + "logps/rejected": -384.5702819824219, + "loss": 0.5406, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.7888541221618652, + "rewards/margins": 0.5834736824035645, + "rewards/rejected": -1.3723278045654297, + "step": 890 + }, + { + "epoch": 0.716, + "grad_norm": 8.971263885498047, + "learning_rate": 1.1310919412686248e-06, + "logits/chosen": -2.5675597190856934, + "logits/rejected": -2.566880464553833, + "logps/chosen": -377.36639404296875, + "logps/rejected": -408.0566711425781, + "loss": 0.5503, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.8293668627738953, + "rewards/margins": 0.5729758739471436, + "rewards/rejected": -1.402342677116394, + "step": 895 + }, + { + "epoch": 0.72, + "grad_norm": 6.206206321716309, + "learning_rate": 1.1020177413231334e-06, + "logits/chosen": -2.5698678493499756, + "logits/rejected": -2.544900417327881, + "logps/chosen": -358.0696716308594, + "logps/rejected": -385.30047607421875, + "loss": 0.5282, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.8030437231063843, + "rewards/margins": 0.6273149251937866, + "rewards/rejected": -1.430358648300171, + "step": 900 + }, + { + "epoch": 0.72, + "eval_logits/chosen": -2.5715651512145996, + "eval_logits/rejected": -2.526627779006958, + "eval_logps/chosen": -362.9278869628906, + "eval_logps/rejected": -390.782470703125, + "eval_loss": 0.541217029094696, + "eval_rewards/accuracies": 0.7083333134651184, + "eval_rewards/chosen": -0.7968972325325012, + "eval_rewards/margins": 0.6306313872337341, + "eval_rewards/rejected": -1.427528738975525, + "eval_runtime": 166.5843, + "eval_samples_per_second": 3.001, + "eval_steps_per_second": 0.378, + "step": 900 + }, + { + "epoch": 0.724, + "grad_norm": 6.695834636688232, + "learning_rate": 1.073216080788921e-06, + "logits/chosen": -2.584725856781006, + "logits/rejected": -2.5650360584259033, + "logps/chosen": -371.1759033203125, + "logps/rejected": -381.4906311035156, + "loss": 0.6249, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.8290435671806335, + "rewards/margins": 0.34561312198638916, + "rewards/rejected": -1.1746567487716675, + "step": 905 + }, + { + "epoch": 0.728, + "grad_norm": 9.597143173217773, + "learning_rate": 1.0446925746067768e-06, + "logits/chosen": -2.5334725379943848, + "logits/rejected": -2.476060152053833, + "logps/chosen": -324.6567687988281, + "logps/rejected": -336.16448974609375, + "loss": 0.5035, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.7902948260307312, + "rewards/margins": 0.6815158724784851, + "rewards/rejected": -1.4718106985092163, + "step": 910 + }, + { + "epoch": 0.732, + "grad_norm": 8.9435396194458, + "learning_rate": 1.0164527834907468e-06, + "logits/chosen": -2.4533045291900635, + "logits/rejected": -2.449897050857544, + "logps/chosen": -347.75250244140625, + "logps/rejected": -426.3138122558594, + "loss": 0.4745, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.8129841685295105, + "rewards/margins": 0.7864343523979187, + "rewards/rejected": -1.5994184017181396, + "step": 915 + }, + { + "epoch": 0.736, + "grad_norm": 9.8597993850708, + "learning_rate": 9.88502212844063e-07, + "logits/chosen": -2.54966139793396, + "logits/rejected": -2.551966428756714, + "logps/chosen": -350.10125732421875, + "logps/rejected": -414.47265625, + "loss": 0.6307, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.8708189129829407, + "rewards/margins": 0.4037759304046631, + "rewards/rejected": -1.274594783782959, + "step": 920 + }, + { + "epoch": 0.74, + "grad_norm": 11.812902450561523, + "learning_rate": 9.608463116858544e-07, + "logits/chosen": -2.5522665977478027, + "logits/rejected": -2.516507625579834, + "logps/chosen": -359.7716064453125, + "logps/rejected": -395.00726318359375, + "loss": 0.5559, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.8488423228263855, + "rewards/margins": 0.5956013798713684, + "rewards/rejected": -1.444443702697754, + "step": 925 + }, + { + "epoch": 0.744, + "grad_norm": 23.08143424987793, + "learning_rate": 9.334904715888496e-07, + "logits/chosen": -2.4799532890319824, + "logits/rejected": -2.480583667755127, + "logps/chosen": -345.7176208496094, + "logps/rejected": -404.23931884765625, + "loss": 0.5426, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.837591826915741, + "rewards/margins": 0.6740916967391968, + "rewards/rejected": -1.5116835832595825, + "step": 930 + }, + { + "epoch": 0.748, + "grad_norm": 9.993616104125977, + "learning_rate": 9.064400256282757e-07, + "logits/chosen": -2.557299852371216, + "logits/rejected": -2.5293102264404297, + "logps/chosen": -361.8759765625, + "logps/rejected": -390.130859375, + "loss": 0.549, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.7642291784286499, + "rewards/margins": 0.6016563773155212, + "rewards/rejected": -1.3658854961395264, + "step": 935 + }, + { + "epoch": 0.752, + "grad_norm": 8.252609252929688, + "learning_rate": 8.797002473421729e-07, + "logits/chosen": -2.5289082527160645, + "logits/rejected": -2.5367226600646973, + "logps/chosen": -387.4449462890625, + "logps/rejected": -411.28057861328125, + "loss": 0.5075, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.6062986254692078, + "rewards/margins": 0.6882797479629517, + "rewards/rejected": -1.2945783138275146, + "step": 940 + }, + { + "epoch": 0.756, + "grad_norm": 15.335317611694336, + "learning_rate": 8.532763497032987e-07, + "logits/chosen": -2.4454190731048584, + "logits/rejected": -2.431666851043701, + "logps/chosen": -370.828857421875, + "logps/rejected": -450.534423828125, + "loss": 0.5028, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.7790817022323608, + "rewards/margins": 0.7363722324371338, + "rewards/rejected": -1.5154539346694946, + "step": 945 + }, + { + "epoch": 0.76, + "grad_norm": 8.324295997619629, + "learning_rate": 8.271734841028553e-07, + "logits/chosen": -2.608813762664795, + "logits/rejected": -2.6157126426696777, + "logps/chosen": -339.1045227050781, + "logps/rejected": -371.37139892578125, + "loss": 0.5261, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.7105227708816528, + "rewards/margins": 0.6277570128440857, + "rewards/rejected": -1.3382797241210938, + "step": 950 + }, + { + "epoch": 0.764, + "grad_norm": 9.136221885681152, + "learning_rate": 8.013967393462094e-07, + "logits/chosen": -2.4687538146972656, + "logits/rejected": -2.490540027618408, + "logps/chosen": -356.60186767578125, + "logps/rejected": -391.22430419921875, + "loss": 0.6001, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.8641288876533508, + "rewards/margins": 0.5564495921134949, + "rewards/rejected": -1.4205783605575562, + "step": 955 + }, + { + "epoch": 0.768, + "grad_norm": 6.628035068511963, + "learning_rate": 7.759511406608255e-07, + "logits/chosen": -2.5683350563049316, + "logits/rejected": -2.500274181365967, + "logps/chosen": -406.158447265625, + "logps/rejected": -411.00360107421875, + "loss": 0.4962, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.9044888615608215, + "rewards/margins": 0.8631958961486816, + "rewards/rejected": -1.7676846981048584, + "step": 960 + }, + { + "epoch": 0.772, + "grad_norm": 8.616596221923828, + "learning_rate": 7.508416487165862e-07, + "logits/chosen": -2.4850573539733887, + "logits/rejected": -2.4977798461914062, + "logps/chosen": -373.7191467285156, + "logps/rejected": -406.65887451171875, + "loss": 0.5838, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.8553136587142944, + "rewards/margins": 0.5251432657241821, + "rewards/rejected": -1.3804569244384766, + "step": 965 + }, + { + "epoch": 0.776, + "grad_norm": 13.198709487915039, + "learning_rate": 7.260731586586983e-07, + "logits/chosen": -2.459974527359009, + "logits/rejected": -2.463366746902466, + "logps/chosen": -344.30938720703125, + "logps/rejected": -415.8106994628906, + "loss": 0.6061, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.9792687296867371, + "rewards/margins": 0.5259648561477661, + "rewards/rejected": -1.505233645439148, + "step": 970 + }, + { + "epoch": 0.78, + "grad_norm": 8.697169303894043, + "learning_rate": 7.016504991533727e-07, + "logits/chosen": -2.5883092880249023, + "logits/rejected": -2.5588154792785645, + "logps/chosen": -385.4810791015625, + "logps/rejected": -427.9122619628906, + "loss": 0.4801, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.6248154044151306, + "rewards/margins": 0.7402738928794861, + "rewards/rejected": -1.3650894165039062, + "step": 975 + }, + { + "epoch": 0.784, + "grad_norm": 5.463360786437988, + "learning_rate": 6.775784314464717e-07, + "logits/chosen": -2.490438461303711, + "logits/rejected": -2.511699914932251, + "logps/chosen": -339.4949645996094, + "logps/rejected": -422.260498046875, + "loss": 0.4897, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.7619988918304443, + "rewards/margins": 0.7652215957641602, + "rewards/rejected": -1.5272204875946045, + "step": 980 + }, + { + "epoch": 0.788, + "grad_norm": 9.091290473937988, + "learning_rate": 6.538616484352902e-07, + "logits/chosen": -2.534930944442749, + "logits/rejected": -2.5236055850982666, + "logps/chosen": -342.08734130859375, + "logps/rejected": -378.9286804199219, + "loss": 0.4952, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.758894145488739, + "rewards/margins": 0.7211824655532837, + "rewards/rejected": -1.480076551437378, + "step": 985 + }, + { + "epoch": 0.792, + "grad_norm": 8.22352123260498, + "learning_rate": 6.305047737536707e-07, + "logits/chosen": -2.502777576446533, + "logits/rejected": -2.4551658630371094, + "logps/chosen": -347.10638427734375, + "logps/rejected": -367.161376953125, + "loss": 0.555, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.8488829731941223, + "rewards/margins": 0.6530700325965881, + "rewards/rejected": -1.501952886581421, + "step": 990 + }, + { + "epoch": 0.796, + "grad_norm": 19.538936614990234, + "learning_rate": 6.075123608706093e-07, + "logits/chosen": -2.542541980743408, + "logits/rejected": -2.5630409717559814, + "logps/chosen": -365.1338806152344, + "logps/rejected": -388.2347106933594, + "loss": 0.5486, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.7868693470954895, + "rewards/margins": 0.5855667591094971, + "rewards/rejected": -1.3724360466003418, + "step": 995 + }, + { + "epoch": 0.8, + "grad_norm": 7.248394012451172, + "learning_rate": 5.848888922025553e-07, + "logits/chosen": -2.4549734592437744, + "logits/rejected": -2.4434893131256104, + "logps/chosen": -330.5204162597656, + "logps/rejected": -420.1793518066406, + "loss": 0.5873, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.9319890141487122, + "rewards/margins": 0.5578263401985168, + "rewards/rejected": -1.4898154735565186, + "step": 1000 + }, + { + "epoch": 0.8, + "eval_logits/chosen": -2.5692970752716064, + "eval_logits/rejected": -2.5253682136535645, + "eval_logps/chosen": -365.5719909667969, + "eval_logps/rejected": -399.30718994140625, + "eval_loss": 0.5368649363517761, + "eval_rewards/accuracies": 0.7083333134651184, + "eval_rewards/chosen": -0.8233387470245361, + "eval_rewards/margins": 0.689436674118042, + "eval_rewards/rejected": -1.5127756595611572, + "eval_runtime": 166.5941, + "eval_samples_per_second": 3.001, + "eval_steps_per_second": 0.378, + "step": 1000 + }, + { + "epoch": 0.804, + "grad_norm": 11.21343994140625, + "learning_rate": 5.626387782395512e-07, + "logits/chosen": -2.5684406757354736, + "logits/rejected": -2.536818265914917, + "logps/chosen": -391.46490478515625, + "logps/rejected": -438.1607971191406, + "loss": 0.5785, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.9650428891181946, + "rewards/margins": 0.6077659726142883, + "rewards/rejected": -1.5728086233139038, + "step": 1005 + }, + { + "epoch": 0.808, + "grad_norm": 7.5349884033203125, + "learning_rate": 5.407663566854008e-07, + "logits/chosen": -2.5107808113098145, + "logits/rejected": -2.4654345512390137, + "logps/chosen": -377.01324462890625, + "logps/rejected": -436.71728515625, + "loss": 0.4949, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.7660607695579529, + "rewards/margins": 0.7943149209022522, + "rewards/rejected": -1.5603755712509155, + "step": 1010 + }, + { + "epoch": 0.812, + "grad_norm": 18.765304565429688, + "learning_rate": 5.192758916120236e-07, + "logits/chosen": -2.5258936882019043, + "logits/rejected": -2.496175527572632, + "logps/chosen": -376.3697509765625, + "logps/rejected": -422.98529052734375, + "loss": 0.5471, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.8881324529647827, + "rewards/margins": 0.6989415884017944, + "rewards/rejected": -1.5870741605758667, + "step": 1015 + }, + { + "epoch": 0.816, + "grad_norm": 9.102749824523926, + "learning_rate": 4.981715726281666e-07, + "logits/chosen": -2.5151665210723877, + "logits/rejected": -2.5129122734069824, + "logps/chosen": -376.29168701171875, + "logps/rejected": -392.8097229003906, + "loss": 0.6469, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.9464821815490723, + "rewards/margins": 0.4086835980415344, + "rewards/rejected": -1.355165958404541, + "step": 1020 + }, + { + "epoch": 0.82, + "grad_norm": 5.625443935394287, + "learning_rate": 4.774575140626317e-07, + "logits/chosen": -2.5486044883728027, + "logits/rejected": -2.55946683883667, + "logps/chosen": -376.15631103515625, + "logps/rejected": -424.7813415527344, + "loss": 0.5013, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.7685045003890991, + "rewards/margins": 0.8368158340454102, + "rewards/rejected": -1.6053203344345093, + "step": 1025 + }, + { + "epoch": 0.824, + "grad_norm": 8.010042190551758, + "learning_rate": 4.5713775416217884e-07, + "logits/chosen": -2.535797357559204, + "logits/rejected": -2.5063552856445312, + "logps/chosen": -369.76129150390625, + "logps/rejected": -407.69281005859375, + "loss": 0.494, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.8518115282058716, + "rewards/margins": 0.8361819982528687, + "rewards/rejected": -1.6879936456680298, + "step": 1030 + }, + { + "epoch": 0.828, + "grad_norm": 10.202120780944824, + "learning_rate": 4.372162543042624e-07, + "logits/chosen": -2.5730769634246826, + "logits/rejected": -2.5308516025543213, + "logps/chosen": -331.8706359863281, + "logps/rejected": -356.06585693359375, + "loss": 0.6335, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.003354787826538, + "rewards/margins": 0.4766755998134613, + "rewards/rejected": -1.4800306558609009, + "step": 1035 + }, + { + "epoch": 0.832, + "grad_norm": 6.7073163986206055, + "learning_rate": 4.1769689822475147e-07, + "logits/chosen": -2.527463912963867, + "logits/rejected": -2.508131980895996, + "logps/chosen": -340.91851806640625, + "logps/rejected": -381.59246826171875, + "loss": 0.5364, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.8549755215644836, + "rewards/margins": 0.6245896220207214, + "rewards/rejected": -1.479565143585205, + "step": 1040 + }, + { + "epoch": 0.836, + "grad_norm": 11.138327598571777, + "learning_rate": 3.9858349126078945e-07, + "logits/chosen": -2.407921314239502, + "logits/rejected": -2.4323601722717285, + "logps/chosen": -367.8681640625, + "logps/rejected": -432.36505126953125, + "loss": 0.5952, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.9345201253890991, + "rewards/margins": 0.5518704652786255, + "rewards/rejected": -1.4863905906677246, + "step": 1045 + }, + { + "epoch": 0.84, + "grad_norm": 18.084115982055664, + "learning_rate": 3.798797596089351e-07, + "logits/chosen": -2.5881738662719727, + "logits/rejected": -2.557568073272705, + "logps/chosen": -387.9930725097656, + "logps/rejected": -412.61773681640625, + "loss": 0.5676, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.9383655786514282, + "rewards/margins": 0.635163426399231, + "rewards/rejected": -1.5735290050506592, + "step": 1050 + }, + { + "epoch": 0.844, + "grad_norm": 8.753804206848145, + "learning_rate": 3.615893495987335e-07, + "logits/chosen": -2.493389129638672, + "logits/rejected": -2.512821912765503, + "logps/chosen": -358.10015869140625, + "logps/rejected": -455.28802490234375, + "loss": 0.5181, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.7339892983436584, + "rewards/margins": 0.7650957703590393, + "rewards/rejected": -1.4990851879119873, + "step": 1055 + }, + { + "epoch": 0.848, + "grad_norm": 10.629037857055664, + "learning_rate": 3.4371582698185636e-07, + "logits/chosen": -2.5065648555755615, + "logits/rejected": -2.5203864574432373, + "logps/chosen": -394.06390380859375, + "logps/rejected": -438.9784240722656, + "loss": 0.4584, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.9274409413337708, + "rewards/margins": 0.83965003490448, + "rewards/rejected": -1.7670910358428955, + "step": 1060 + }, + { + "epoch": 0.852, + "grad_norm": 12.203771591186523, + "learning_rate": 3.262626762369525e-07, + "logits/chosen": -2.5433928966522217, + "logits/rejected": -2.4640583992004395, + "logps/chosen": -335.0285339355469, + "logps/rejected": -358.05206298828125, + "loss": 0.5235, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.8205272555351257, + "rewards/margins": 0.7125735282897949, + "rewards/rejected": -1.5331008434295654, + "step": 1065 + }, + { + "epoch": 0.856, + "grad_norm": 10.726627349853516, + "learning_rate": 3.092332998903416e-07, + "logits/chosen": -2.5511505603790283, + "logits/rejected": -2.555680990219116, + "logps/chosen": -387.1297912597656, + "logps/rejected": -438.8545837402344, + "loss": 0.5697, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.7436598539352417, + "rewards/margins": 0.5584943890571594, + "rewards/rejected": -1.302154302597046, + "step": 1070 + }, + { + "epoch": 0.86, + "grad_norm": 10.780182838439941, + "learning_rate": 2.9263101785268253e-07, + "logits/chosen": -2.5446937084198, + "logits/rejected": -2.5189390182495117, + "logps/chosen": -376.8099670410156, + "logps/rejected": -390.14105224609375, + "loss": 0.6353, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.9379167556762695, + "rewards/margins": 0.45755672454833984, + "rewards/rejected": -1.3954734802246094, + "step": 1075 + }, + { + "epoch": 0.864, + "grad_norm": 9.165637969970703, + "learning_rate": 2.764590667717562e-07, + "logits/chosen": -2.520571231842041, + "logits/rejected": -2.5012898445129395, + "logps/chosen": -348.20050048828125, + "logps/rejected": -430.0379333496094, + "loss": 0.4818, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.767501175403595, + "rewards/margins": 0.8683005571365356, + "rewards/rejected": -1.6358016729354858, + "step": 1080 + }, + { + "epoch": 0.868, + "grad_norm": 15.374898910522461, + "learning_rate": 2.6072059940146775e-07, + "logits/chosen": -2.4875528812408447, + "logits/rejected": -2.4620859622955322, + "logps/chosen": -358.6385192871094, + "logps/rejected": -375.6526794433594, + "loss": 0.641, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.9842912554740906, + "rewards/margins": 0.40321025252342224, + "rewards/rejected": -1.3875017166137695, + "step": 1085 + }, + { + "epoch": 0.872, + "grad_norm": 16.681074142456055, + "learning_rate": 2.454186839872158e-07, + "logits/chosen": -2.4695944786071777, + "logits/rejected": -2.4315385818481445, + "logps/chosen": -369.12884521484375, + "logps/rejected": -432.47467041015625, + "loss": 0.5636, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.8553116917610168, + "rewards/margins": 0.5986126065254211, + "rewards/rejected": -1.453924298286438, + "step": 1090 + }, + { + "epoch": 0.876, + "grad_norm": 12.826825141906738, + "learning_rate": 2.3055630366772857e-07, + "logits/chosen": -2.560246229171753, + "logits/rejected": -2.5458738803863525, + "logps/chosen": -357.64227294921875, + "logps/rejected": -396.8709411621094, + "loss": 0.5257, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.7637326717376709, + "rewards/margins": 0.737055242061615, + "rewards/rejected": -1.5007880926132202, + "step": 1095 + }, + { + "epoch": 0.88, + "grad_norm": 10.747795104980469, + "learning_rate": 2.1613635589349756e-07, + "logits/chosen": -2.552135944366455, + "logits/rejected": -2.548779010772705, + "logps/chosen": -344.90142822265625, + "logps/rejected": -395.3324279785156, + "loss": 0.5152, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.8041820526123047, + "rewards/margins": 0.7682543396949768, + "rewards/rejected": -1.5724363327026367, + "step": 1100 + }, + { + "epoch": 0.88, + "eval_logits/chosen": -2.5619542598724365, + "eval_logits/rejected": -2.518826723098755, + "eval_logps/chosen": -357.70245361328125, + "eval_logps/rejected": -389.9855041503906, + "eval_loss": 0.5383636951446533, + "eval_rewards/accuracies": 0.7142857313156128, + "eval_rewards/chosen": -0.7446432113647461, + "eval_rewards/margins": 0.6749160289764404, + "eval_rewards/rejected": -1.4195590019226074, + "eval_runtime": 166.7191, + "eval_samples_per_second": 2.999, + "eval_steps_per_second": 0.378, + "step": 1100 + }, + { + "epoch": 0.884, + "grad_norm": 15.551572799682617, + "learning_rate": 2.0216165186191406e-07, + "logits/chosen": -2.529543399810791, + "logits/rejected": -2.511012554168701, + "logps/chosen": -359.8935546875, + "logps/rejected": -426.3463439941406, + "loss": 0.5322, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.7537140846252441, + "rewards/margins": 0.747148871421814, + "rewards/rejected": -1.5008628368377686, + "step": 1105 + }, + { + "epoch": 0.888, + "grad_norm": 10.654102325439453, + "learning_rate": 1.8863491596921745e-07, + "logits/chosen": -2.531877040863037, + "logits/rejected": -2.5003228187561035, + "logps/chosen": -394.6542053222656, + "logps/rejected": -419.4615173339844, + "loss": 0.6303, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.9292774200439453, + "rewards/margins": 0.46585217118263245, + "rewards/rejected": -1.3951294422149658, + "step": 1110 + }, + { + "epoch": 0.892, + "grad_norm": 12.998001098632812, + "learning_rate": 1.7555878527937164e-07, + "logits/chosen": -2.6154770851135254, + "logits/rejected": -2.573152542114258, + "logps/chosen": -378.4177551269531, + "logps/rejected": -407.0361328125, + "loss": 0.4692, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.8625243902206421, + "rewards/margins": 0.8568865060806274, + "rewards/rejected": -1.7194106578826904, + "step": 1115 + }, + { + "epoch": 0.896, + "grad_norm": 9.31609058380127, + "learning_rate": 1.629358090099639e-07, + "logits/chosen": -2.4966423511505127, + "logits/rejected": -2.4891440868377686, + "logps/chosen": -389.1834411621094, + "logps/rejected": -428.36199951171875, + "loss": 0.5027, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.8896617889404297, + "rewards/margins": 0.7314151525497437, + "rewards/rejected": -1.6210769414901733, + "step": 1120 + }, + { + "epoch": 0.9, + "grad_norm": 10.968253135681152, + "learning_rate": 1.507684480352292e-07, + "logits/chosen": -2.5237255096435547, + "logits/rejected": -2.5314764976501465, + "logps/chosen": -367.54150390625, + "logps/rejected": -418.0208435058594, + "loss": 0.5305, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.9517295956611633, + "rewards/margins": 0.6929836869239807, + "rewards/rejected": -1.6447131633758545, + "step": 1125 + }, + { + "epoch": 0.904, + "grad_norm": 9.858650207519531, + "learning_rate": 1.3905907440629752e-07, + "logits/chosen": -2.551881790161133, + "logits/rejected": -2.5350139141082764, + "logps/chosen": -366.10174560546875, + "logps/rejected": -396.09478759765625, + "loss": 0.551, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.8920289874076843, + "rewards/margins": 0.6426266431808472, + "rewards/rejected": -1.5346556901931763, + "step": 1130 + }, + { + "epoch": 0.908, + "grad_norm": 9.662504196166992, + "learning_rate": 1.278099708887587e-07, + "logits/chosen": -2.5561671257019043, + "logits/rejected": -2.5373544692993164, + "logps/chosen": -344.9742431640625, + "logps/rejected": -458.02313232421875, + "loss": 0.5248, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.779849112033844, + "rewards/margins": 0.7551368474960327, + "rewards/rejected": -1.534985899925232, + "step": 1135 + }, + { + "epoch": 0.912, + "grad_norm": 12.876849174499512, + "learning_rate": 1.1702333051763271e-07, + "logits/chosen": -2.5694494247436523, + "logits/rejected": -2.5618202686309814, + "logps/chosen": -399.39739990234375, + "logps/rejected": -404.67535400390625, + "loss": 0.541, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.8758378028869629, + "rewards/margins": 0.7352150082588196, + "rewards/rejected": -1.6110527515411377, + "step": 1140 + }, + { + "epoch": 0.916, + "grad_norm": 10.277182579040527, + "learning_rate": 1.067012561698319e-07, + "logits/chosen": -2.5358214378356934, + "logits/rejected": -2.5226242542266846, + "logps/chosen": -378.08331298828125, + "logps/rejected": -406.0291442871094, + "loss": 0.6472, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.8850030899047852, + "rewards/margins": 0.44015589356422424, + "rewards/rejected": -1.3251588344573975, + "step": 1145 + }, + { + "epoch": 0.92, + "grad_norm": 17.924530029296875, + "learning_rate": 9.684576015420277e-08, + "logits/chosen": -2.4903244972229004, + "logits/rejected": -2.4617245197296143, + "logps/chosen": -328.5221252441406, + "logps/rejected": -361.15728759765625, + "loss": 0.5055, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.7505359053611755, + "rewards/margins": 0.7073702812194824, + "rewards/rejected": -1.4579061269760132, + "step": 1150 + }, + { + "epoch": 0.924, + "grad_norm": 14.877918243408203, + "learning_rate": 8.745876381922147e-08, + "logits/chosen": -2.4926960468292236, + "logits/rejected": -2.525510549545288, + "logps/chosen": -342.7877197265625, + "logps/rejected": -372.467041015625, + "loss": 0.5811, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.8225164413452148, + "rewards/margins": 0.6331211924552917, + "rewards/rejected": -1.4556376934051514, + "step": 1155 + }, + { + "epoch": 0.928, + "grad_norm": 14.825825691223145, + "learning_rate": 7.854209717842231e-08, + "logits/chosen": -2.560148239135742, + "logits/rejected": -2.536252975463867, + "logps/chosen": -389.2018127441406, + "logps/rejected": -387.626220703125, + "loss": 0.6612, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -0.9954813122749329, + "rewards/margins": 0.3569512963294983, + "rewards/rejected": -1.3524326086044312, + "step": 1160 + }, + { + "epoch": 0.932, + "grad_norm": 14.72794246673584, + "learning_rate": 7.009749855363457e-08, + "logits/chosen": -2.5346550941467285, + "logits/rejected": -2.5153279304504395, + "logps/chosen": -340.6593322753906, + "logps/rejected": -409.87274169921875, + "loss": 0.5073, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.70791095495224, + "rewards/margins": 0.6868191361427307, + "rewards/rejected": -1.3947298526763916, + "step": 1165 + }, + { + "epoch": 0.936, + "grad_norm": 12.192831993103027, + "learning_rate": 6.212661423609184e-08, + "logits/chosen": -2.601175308227539, + "logits/rejected": -2.5409445762634277, + "logps/chosen": -391.4974060058594, + "logps/rejected": -429.298583984375, + "loss": 0.5721, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.9178189039230347, + "rewards/margins": 0.643661379814148, + "rewards/rejected": -1.5614804029464722, + "step": 1170 + }, + { + "epoch": 0.94, + "grad_norm": 16.621145248413086, + "learning_rate": 5.463099816548578e-08, + "logits/chosen": -2.517141819000244, + "logits/rejected": -2.512964963912964, + "logps/chosen": -360.8076171875, + "logps/rejected": -446.977294921875, + "loss": 0.5067, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.9305839538574219, + "rewards/margins": 0.7509930729866028, + "rewards/rejected": -1.6815770864486694, + "step": 1175 + }, + { + "epoch": 0.944, + "grad_norm": 9.267435073852539, + "learning_rate": 4.761211162702117e-08, + "logits/chosen": -2.5712485313415527, + "logits/rejected": -2.508033037185669, + "logps/chosen": -396.81658935546875, + "logps/rejected": -446.858154296875, + "loss": 0.5266, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.7593949437141418, + "rewards/margins": 0.6189205050468445, + "rewards/rejected": -1.3783155679702759, + "step": 1180 + }, + { + "epoch": 0.948, + "grad_norm": 6.9370436668396, + "learning_rate": 4.1071322966535487e-08, + "logits/chosen": -2.588221788406372, + "logits/rejected": -2.5175411701202393, + "logps/chosen": -418.30474853515625, + "logps/rejected": -404.17840576171875, + "loss": 0.4887, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.7452821135520935, + "rewards/margins": 0.8689033389091492, + "rewards/rejected": -1.6141853332519531, + "step": 1185 + }, + { + "epoch": 0.952, + "grad_norm": 4.675684452056885, + "learning_rate": 3.5009907323737826e-08, + "logits/chosen": -2.5127742290496826, + "logits/rejected": -2.58686900138855, + "logps/chosen": -371.75390625, + "logps/rejected": -481.8087463378906, + "loss": 0.4452, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.6970502138137817, + "rewards/margins": 0.9857767820358276, + "rewards/rejected": -1.6828269958496094, + "step": 1190 + }, + { + "epoch": 0.956, + "grad_norm": 8.315820693969727, + "learning_rate": 2.9429046383618042e-08, + "logits/chosen": -2.4666857719421387, + "logits/rejected": -2.4609508514404297, + "logps/chosen": -369.11480712890625, + "logps/rejected": -399.99835205078125, + "loss": 0.4684, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.7124398946762085, + "rewards/margins": 0.7592355608940125, + "rewards/rejected": -1.4716756343841553, + "step": 1195 + }, + { + "epoch": 0.96, + "grad_norm": 13.105049133300781, + "learning_rate": 2.4329828146074096e-08, + "logits/chosen": -2.531951904296875, + "logits/rejected": -2.5043699741363525, + "logps/chosen": -376.4866943359375, + "logps/rejected": -373.62109375, + "loss": 0.5213, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.8802105188369751, + "rewards/margins": 0.7113884091377258, + "rewards/rejected": -1.5915989875793457, + "step": 1200 + }, + { + "epoch": 0.96, + "eval_logits/chosen": -2.556800365447998, + "eval_logits/rejected": -2.513496160507202, + "eval_logps/chosen": -362.12188720703125, + "eval_logps/rejected": -395.5133361816406, + "eval_loss": 0.5369879007339478, + "eval_rewards/accuracies": 0.7063491940498352, + "eval_rewards/chosen": -0.7888382077217102, + "eval_rewards/margins": 0.6859992146492004, + "eval_rewards/rejected": -1.4748374223709106, + "eval_runtime": 166.4514, + "eval_samples_per_second": 3.004, + "eval_steps_per_second": 0.378, + "step": 1200 + }, + { + "epoch": 0.964, + "grad_norm": 8.366193771362305, + "learning_rate": 1.9713246713805588e-08, + "logits/chosen": -2.4164879322052, + "logits/rejected": -2.395017623901367, + "logps/chosen": -332.5264892578125, + "logps/rejected": -406.001953125, + "loss": 0.4566, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.618050217628479, + "rewards/margins": 0.9078122973442078, + "rewards/rejected": -1.525862455368042, + "step": 1205 + }, + { + "epoch": 0.968, + "grad_norm": 9.794474601745605, + "learning_rate": 1.5580202098509078e-08, + "logits/chosen": -2.498256206512451, + "logits/rejected": -2.455946207046509, + "logps/chosen": -413.91802978515625, + "logps/rejected": -458.69671630859375, + "loss": 0.6104, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.9604536890983582, + "rewards/margins": 0.4910499155521393, + "rewards/rejected": -1.4515035152435303, + "step": 1210 + }, + { + "epoch": 0.972, + "grad_norm": 9.097041130065918, + "learning_rate": 1.193150004542204e-08, + "logits/chosen": -2.531789779663086, + "logits/rejected": -2.5278313159942627, + "logps/chosen": -356.40386962890625, + "logps/rejected": -406.5818786621094, + "loss": 0.5828, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.6918624639511108, + "rewards/margins": 0.5878754258155823, + "rewards/rejected": -1.2797380685806274, + "step": 1215 + }, + { + "epoch": 0.976, + "grad_norm": 10.505363464355469, + "learning_rate": 8.767851876239075e-09, + "logits/chosen": -2.512781858444214, + "logits/rejected": -2.4617748260498047, + "logps/chosen": -325.30657958984375, + "logps/rejected": -373.95013427734375, + "loss": 0.5752, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.7784557342529297, + "rewards/margins": 0.626113772392273, + "rewards/rejected": -1.4045695066452026, + "step": 1220 + }, + { + "epoch": 0.98, + "grad_norm": 9.12982177734375, + "learning_rate": 6.089874350439507e-09, + "logits/chosen": -2.5082201957702637, + "logits/rejected": -2.4916832447052, + "logps/chosen": -437.98638916015625, + "logps/rejected": -452.052001953125, + "loss": 0.5096, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.8293756246566772, + "rewards/margins": 0.6941211819648743, + "rewards/rejected": -1.5234968662261963, + "step": 1225 + }, + { + "epoch": 0.984, + "grad_norm": 9.886011123657227, + "learning_rate": 3.8980895450474455e-09, + "logits/chosen": -2.47562313079834, + "logits/rejected": -2.472003221511841, + "logps/chosen": -375.73370361328125, + "logps/rejected": -489.0946350097656, + "loss": 0.4262, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.7051881551742554, + "rewards/margins": 0.9877891540527344, + "rewards/rejected": -1.6929775476455688, + "step": 1230 + }, + { + "epoch": 0.988, + "grad_norm": 8.874799728393555, + "learning_rate": 2.192924752854042e-09, + "logits/chosen": -2.579641819000244, + "logits/rejected": -2.561047077178955, + "logps/chosen": -357.88421630859375, + "logps/rejected": -407.43438720703125, + "loss": 0.5599, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.7861407995223999, + "rewards/margins": 0.5933399796485901, + "rewards/rejected": -1.3794807195663452, + "step": 1235 + }, + { + "epoch": 0.992, + "grad_norm": 8.425810813903809, + "learning_rate": 9.747123991141193e-10, + "logits/chosen": -2.442471981048584, + "logits/rejected": -2.426997661590576, + "logps/chosen": -376.2283020019531, + "logps/rejected": -400.55718994140625, + "loss": 0.575, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9100795984268188, + "rewards/margins": 0.6220163702964783, + "rewards/rejected": -1.5320959091186523, + "step": 1240 + }, + { + "epoch": 0.996, + "grad_norm": 9.747213363647461, + "learning_rate": 2.43689976739403e-10, + "logits/chosen": -2.400465250015259, + "logits/rejected": -2.4487109184265137, + "logps/chosen": -408.04913330078125, + "logps/rejected": -413.04443359375, + "loss": 0.5348, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.8299952745437622, + "rewards/margins": 0.6040025353431702, + "rewards/rejected": -1.4339977502822876, + "step": 1245 + }, + { + "epoch": 1.0, + "grad_norm": 9.561023712158203, + "learning_rate": 0.0, + "logits/chosen": -2.478743076324463, + "logits/rejected": -2.454713821411133, + "logps/chosen": -397.37994384765625, + "logps/rejected": -446.76275634765625, + "loss": 0.5129, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.9160435795783997, + "rewards/margins": 0.6437736749649048, + "rewards/rejected": -1.5598171949386597, + "step": 1250 + }, + { + "epoch": 1.0, + "step": 1250, + "total_flos": 0.0, + "train_loss": 0.5874967678070069, + "train_runtime": 15864.6959, + "train_samples_per_second": 1.261, + "train_steps_per_second": 0.079 + } + ], + "logging_steps": 5, + "max_steps": 1250, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}