| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9984301412872841, | |
| "eval_steps": 500, | |
| "global_step": 159, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.006279434850863423, | |
| "grad_norm": 17.524188433318294, | |
| "learning_rate": 1.875e-08, | |
| "logits/chosen": 0.050171270966529846, | |
| "logits/rejected": 0.7975481748580933, | |
| "logps/chosen": -213.99826049804688, | |
| "logps/pi_response": -122.33531951904297, | |
| "logps/ref_response": -122.33531951904297, | |
| "logps/rejected": -327.0420227050781, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.06279434850863422, | |
| "grad_norm": 23.45193685304271, | |
| "learning_rate": 1.875e-07, | |
| "logits/chosen": 0.6251156330108643, | |
| "logits/rejected": 0.9096591472625732, | |
| "logps/chosen": -268.11822509765625, | |
| "logps/pi_response": -114.67009735107422, | |
| "logps/ref_response": -114.69570922851562, | |
| "logps/rejected": -390.1163330078125, | |
| "loss": 0.6924, | |
| "rewards/accuracies": 0.5416666865348816, | |
| "rewards/chosen": -0.000606474990490824, | |
| "rewards/margins": 0.0019376208074390888, | |
| "rewards/rejected": -0.0025440959725528955, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.12558869701726844, | |
| "grad_norm": 19.551172670207706, | |
| "learning_rate": 2.9942119880575817e-07, | |
| "logits/chosen": 0.4843064248561859, | |
| "logits/rejected": 0.8785603642463684, | |
| "logps/chosen": -269.538330078125, | |
| "logps/pi_response": -121.35018157958984, | |
| "logps/ref_response": -121.34931945800781, | |
| "logps/rejected": -417.247314453125, | |
| "loss": 0.6747, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.0313839390873909, | |
| "rewards/margins": 0.04394400864839554, | |
| "rewards/rejected": -0.07532794773578644, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.18838304552590268, | |
| "grad_norm": 13.701658129429845, | |
| "learning_rate": 2.929608750821129e-07, | |
| "logits/chosen": 0.5343824625015259, | |
| "logits/rejected": 0.9908145666122437, | |
| "logps/chosen": -310.638671875, | |
| "logps/pi_response": -118.42143249511719, | |
| "logps/ref_response": -117.36665344238281, | |
| "logps/rejected": -428.20166015625, | |
| "loss": 0.6175, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.18561159074306488, | |
| "rewards/margins": 0.17133654654026031, | |
| "rewards/rejected": -0.3569481372833252, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.25117739403453687, | |
| "grad_norm": 14.693007375551543, | |
| "learning_rate": 2.7962832564252725e-07, | |
| "logits/chosen": 0.5080328583717346, | |
| "logits/rejected": 0.9311200976371765, | |
| "logps/chosen": -303.4456481933594, | |
| "logps/pi_response": -121.04774475097656, | |
| "logps/ref_response": -117.7525634765625, | |
| "logps/rejected": -484.09552001953125, | |
| "loss": 0.5785, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.3230215311050415, | |
| "rewards/margins": 0.4571772515773773, | |
| "rewards/rejected": -0.7801988124847412, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.3139717425431711, | |
| "grad_norm": 18.087271424489508, | |
| "learning_rate": 2.6006445513357056e-07, | |
| "logits/chosen": 0.6870445013046265, | |
| "logits/rejected": 0.9769166111946106, | |
| "logps/chosen": -342.7091369628906, | |
| "logps/pi_response": -124.7760009765625, | |
| "logps/ref_response": -115.9255599975586, | |
| "logps/rejected": -495.4518127441406, | |
| "loss": 0.5714, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.712685227394104, | |
| "rewards/margins": 0.5671727657318115, | |
| "rewards/rejected": -1.2798579931259155, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.37676609105180536, | |
| "grad_norm": 11.166951031052399, | |
| "learning_rate": 2.3520971200967334e-07, | |
| "logits/chosen": 0.5992540121078491, | |
| "logits/rejected": 1.020711898803711, | |
| "logps/chosen": -348.0439758300781, | |
| "logps/pi_response": -127.2256088256836, | |
| "logps/ref_response": -119.3854751586914, | |
| "logps/rejected": -496.8860778808594, | |
| "loss": 0.5649, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -0.5891093015670776, | |
| "rewards/margins": 0.5816227793693542, | |
| "rewards/rejected": -1.1707321405410767, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.43956043956043955, | |
| "grad_norm": 11.470597805963651, | |
| "learning_rate": 2.0625888054143427e-07, | |
| "logits/chosen": 0.6002563238143921, | |
| "logits/rejected": 0.9772939682006836, | |
| "logps/chosen": -272.6213073730469, | |
| "logps/pi_response": -127.3548583984375, | |
| "logps/ref_response": -120.81624603271484, | |
| "logps/rejected": -524.6642456054688, | |
| "loss": 0.5372, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.36886435747146606, | |
| "rewards/margins": 0.7560935020446777, | |
| "rewards/rejected": -1.124957799911499, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.5023547880690737, | |
| "grad_norm": 11.00220562972207, | |
| "learning_rate": 1.7460364672965327e-07, | |
| "logits/chosen": 0.6907710433006287, | |
| "logits/rejected": 1.0923134088516235, | |
| "logps/chosen": -269.1040954589844, | |
| "logps/pi_response": -112.93983459472656, | |
| "logps/ref_response": -106.69105529785156, | |
| "logps/rejected": -492.8182678222656, | |
| "loss": 0.5271, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.4200579524040222, | |
| "rewards/margins": 0.6648958921432495, | |
| "rewards/rejected": -1.084953784942627, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.565149136577708, | |
| "grad_norm": 11.111343223576297, | |
| "learning_rate": 1.4176569902035086e-07, | |
| "logits/chosen": 0.6897394061088562, | |
| "logits/rejected": 1.039529800415039, | |
| "logps/chosen": -341.63275146484375, | |
| "logps/pi_response": -120.40811920166016, | |
| "logps/ref_response": -111.32686614990234, | |
| "logps/rejected": -511.60565185546875, | |
| "loss": 0.5055, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.5421563982963562, | |
| "rewards/margins": 0.7095355987548828, | |
| "rewards/rejected": -1.2516921758651733, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.6279434850863422, | |
| "grad_norm": 12.249034688194289, | |
| "learning_rate": 1.0932357971453743e-07, | |
| "logits/chosen": 0.7736852765083313, | |
| "logits/rejected": 1.0484180450439453, | |
| "logps/chosen": -290.811279296875, | |
| "logps/pi_response": -116.5997543334961, | |
| "logps/ref_response": -107.03324127197266, | |
| "logps/rejected": -533.04931640625, | |
| "loss": 0.539, | |
| "rewards/accuracies": 0.7562500238418579, | |
| "rewards/chosen": -0.49749264121055603, | |
| "rewards/margins": 0.8334757089614868, | |
| "rewards/rejected": -1.3309683799743652, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.6907378335949764, | |
| "grad_norm": 10.864663994027794, | |
| "learning_rate": 7.883680337481599e-08, | |
| "logits/chosen": 0.7670079469680786, | |
| "logits/rejected": 0.9956636428833008, | |
| "logps/chosen": -305.14593505859375, | |
| "logps/pi_response": -127.0176773071289, | |
| "logps/ref_response": -116.74520111083984, | |
| "logps/rejected": -521.2225341796875, | |
| "loss": 0.5335, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.5058807134628296, | |
| "rewards/margins": 0.7484409809112549, | |
| "rewards/rejected": -1.2543216943740845, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.7535321821036107, | |
| "grad_norm": 11.061608119367962, | |
| "learning_rate": 5.177088990820725e-08, | |
| "logits/chosen": 0.5085455775260925, | |
| "logits/rejected": 0.8686326146125793, | |
| "logps/chosen": -308.0654602050781, | |
| "logps/pi_response": -132.87335205078125, | |
| "logps/ref_response": -123.3708267211914, | |
| "logps/rejected": -556.9476928710938, | |
| "loss": 0.5259, | |
| "rewards/accuracies": 0.7562500238418579, | |
| "rewards/chosen": -0.4727388918399811, | |
| "rewards/margins": 0.8706821203231812, | |
| "rewards/rejected": -1.3434208631515503, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.8163265306122449, | |
| "grad_norm": 13.028590054698721, | |
| "learning_rate": 2.942691603548416e-08, | |
| "logits/chosen": 0.5405811071395874, | |
| "logits/rejected": 1.0355885028839111, | |
| "logps/chosen": -320.9105224609375, | |
| "logps/pi_response": -135.04757690429688, | |
| "logps/ref_response": -127.4443588256836, | |
| "logps/rejected": -553.2946166992188, | |
| "loss": 0.5056, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.4617777466773987, | |
| "rewards/margins": 0.8675910234451294, | |
| "rewards/rejected": -1.3293688297271729, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.8791208791208791, | |
| "grad_norm": 11.32561508697925, | |
| "learning_rate": 1.2878971655412513e-08, | |
| "logits/chosen": 0.5967448353767395, | |
| "logits/rejected": 0.9229636192321777, | |
| "logps/chosen": -296.9305725097656, | |
| "logps/pi_response": -139.7073974609375, | |
| "logps/ref_response": -129.02000427246094, | |
| "logps/rejected": -566.9810791015625, | |
| "loss": 0.5117, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.4712587893009186, | |
| "rewards/margins": 0.9010453224182129, | |
| "rewards/rejected": -1.3723042011260986, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.9419152276295133, | |
| "grad_norm": 11.652819230057418, | |
| "learning_rate": 2.922527618666465e-09, | |
| "logits/chosen": 0.5724425315856934, | |
| "logits/rejected": 0.9267145395278931, | |
| "logps/chosen": -312.5321350097656, | |
| "logps/pi_response": -124.59476470947266, | |
| "logps/ref_response": -114.87628173828125, | |
| "logps/rejected": -503.21697998046875, | |
| "loss": 0.5304, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.5373696088790894, | |
| "rewards/margins": 0.6669692993164062, | |
| "rewards/rejected": -1.204338788986206, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.9984301412872841, | |
| "step": 159, | |
| "total_flos": 0.0, | |
| "train_loss": 0.557155561147246, | |
| "train_runtime": 4462.021, | |
| "train_samples_per_second": 4.567, | |
| "train_steps_per_second": 0.036 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 159, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |