{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 91,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "debug/policy_chosen_logits": 22.537443161010742,
      "debug/policy_chosen_logps": -454.7864685058594,
      "debug/policy_rejected_logits": 24.08443260192871,
      "debug/policy_rejected_logps": -485.6905517578125,
      "debug/reference_chosen_logps": -454.7864685058594,
      "debug/reference_rejected_logps": -485.6905517578125,
      "epoch": 0.01098901098901099,
      "grad_norm": 6.833481499698497,
      "learning_rate": 1e-06,
      "logits/chosen": 22.537443161010742,
      "logits/rejected": 24.08443260192871,
      "logps/chosen": -454.7864685058594,
      "logps/rejected": -485.6905517578125,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "debug/policy_chosen_logits": 25.2491512298584,
      "debug/policy_chosen_logps": -426.63800048828125,
      "debug/policy_rejected_logits": 25.061065673828125,
      "debug/policy_rejected_logps": -446.8348083496094,
      "debug/reference_chosen_logps": -426.80908203125,
      "debug/reference_rejected_logps": -446.17877197265625,
      "epoch": 0.02197802197802198,
      "grad_norm": 8.518360012009484,
      "learning_rate": 1e-06,
      "logits/chosen": 25.2491512298584,
      "logits/rejected": 25.061065673828125,
      "logps/chosen": -426.63800048828125,
      "logps/rejected": -446.8348083496094,
      "loss": 0.5003,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0017108535394072533,
      "rewards/margins": 0.008270950056612492,
      "rewards/rejected": -0.006560096517205238,
      "step": 2
    },
    {
      "debug/policy_chosen_logits": 23.04501724243164,
      "debug/policy_chosen_logps": -418.5323791503906,
      "debug/policy_rejected_logits": 26.21137809753418,
      "debug/policy_rejected_logps": -445.2972412109375,
      "debug/reference_chosen_logps": -418.4830627441406,
      "debug/reference_rejected_logps": -445.174560546875,
      "epoch": 0.03296703296703297,
      "grad_norm": 8.234274066877381,
      "learning_rate": 1e-06,
      "logits/chosen": 23.04501724243164,
      "logits/rejected": 26.21137809753418,
      "logps/chosen": -418.5323791503906,
      "logps/rejected": -445.2972412109375,
      "loss": 0.497,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0004932782612740993,
      "rewards/margins": 0.0007336426060646772,
      "rewards/rejected": -0.001226921333000064,
      "step": 3
    },
    {
      "debug/policy_chosen_logits": 22.864124298095703,
      "debug/policy_chosen_logps": -424.31500244140625,
      "debug/policy_rejected_logits": 24.804826736450195,
      "debug/policy_rejected_logps": -440.4050598144531,
      "debug/reference_chosen_logps": -424.2529296875,
      "debug/reference_rejected_logps": -440.0723876953125,
      "epoch": 0.04395604395604396,
      "grad_norm": 8.267600116778256,
      "learning_rate": 1e-06,
      "logits/chosen": 22.864124298095703,
      "logits/rejected": 24.804826736450195,
      "logps/chosen": -424.31500244140625,
      "logps/rejected": -440.4050598144531,
      "loss": 0.4948,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.000620613107457757,
      "rewards/margins": 0.0027059551794081926,
      "rewards/rejected": -0.0033265682868659496,
      "step": 4
    },
    {
      "debug/policy_chosen_logits": 23.74032211303711,
      "debug/policy_chosen_logps": -425.2148132324219,
      "debug/policy_rejected_logits": 24.66983985900879,
      "debug/policy_rejected_logps": -428.9949951171875,
      "debug/reference_chosen_logps": -425.707763671875,
      "debug/reference_rejected_logps": -427.61578369140625,
      "epoch": 0.054945054945054944,
      "grad_norm": 7.1377920370536465,
      "learning_rate": 1e-06,
      "logits/chosen": 23.74032211303711,
      "logits/rejected": 24.66983985900879,
      "logps/chosen": -425.2148132324219,
      "logps/rejected": -428.9949951171875,
      "loss": 0.4915,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.004929542541503906,
      "rewards/margins": 0.018722152337431908,
      "rewards/rejected": -0.013792609795928001,
      "step": 5
    },
    {
      "debug/policy_chosen_logits": 22.908384323120117,
      "debug/policy_chosen_logps": -436.1788635253906,
      "debug/policy_rejected_logits": 26.066926956176758,
      "debug/policy_rejected_logps": -458.21295166015625,
      "debug/reference_chosen_logps": -437.0372619628906,
      "debug/reference_rejected_logps": -455.97064208984375,
      "epoch": 0.06593406593406594,
      "grad_norm": 6.840813228978009,
      "learning_rate": 1e-06,
      "logits/chosen": 22.908384323120117,
      "logits/rejected": 26.066926956176758,
      "logps/chosen": -436.1788635253906,
      "logps/rejected": -458.21295166015625,
      "loss": 0.4897,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008583908900618553,
      "rewards/margins": 0.031006699427962303,
      "rewards/rejected": -0.02242279052734375,
      "step": 6
    },
    {
      "debug/policy_chosen_logits": 27.81578826904297,
      "debug/policy_chosen_logps": -456.6412353515625,
      "debug/policy_rejected_logits": 22.0266170501709,
      "debug/policy_rejected_logps": -461.65155029296875,
      "debug/reference_chosen_logps": -457.8715515136719,
      "debug/reference_rejected_logps": -459.9510498046875,
      "epoch": 0.07692307692307693,
      "grad_norm": 7.815334550310639,
      "learning_rate": 1e-06,
      "logits/chosen": 27.81578826904297,
      "logits/rejected": 22.0266170501709,
      "logps/chosen": -456.6412353515625,
      "logps/rejected": -461.65155029296875,
      "loss": 0.4758,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.012303275987505913,
      "rewards/margins": 0.029308240860700607,
      "rewards/rejected": -0.017004966735839844,
      "step": 7
    },
    {
      "debug/policy_chosen_logits": 21.05910301208496,
      "debug/policy_chosen_logps": -420.5210876464844,
      "debug/policy_rejected_logits": 21.690637588500977,
      "debug/policy_rejected_logps": -461.24737548828125,
      "debug/reference_chosen_logps": -421.00885009765625,
      "debug/reference_rejected_logps": -459.6142578125,
      "epoch": 0.08791208791208792,
      "grad_norm": 6.940663555739121,
      "learning_rate": 1e-06,
      "logits/chosen": 21.05910301208496,
      "logits/rejected": 21.690637588500977,
      "logps/chosen": -420.5210876464844,
      "logps/rejected": -461.24737548828125,
      "loss": 0.4748,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004877815023064613,
      "rewards/margins": 0.02120864763855934,
      "rewards/rejected": -0.016330832615494728,
      "step": 8
    },
    {
      "debug/policy_chosen_logits": 25.456287384033203,
      "debug/policy_chosen_logps": -440.22540283203125,
      "debug/policy_rejected_logits": 26.438932418823242,
      "debug/policy_rejected_logps": -419.1558837890625,
      "debug/reference_chosen_logps": -439.4281921386719,
      "debug/reference_rejected_logps": -417.99652099609375,
      "epoch": 0.0989010989010989,
      "grad_norm": 5.7812586478558865,
      "learning_rate": 1e-06,
      "logits/chosen": 25.456287384033203,
      "logits/rejected": 26.438932418823242,
      "logps/chosen": -440.22540283203125,
      "logps/rejected": -419.1558837890625,
      "loss": 0.4769,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007972106337547302,
      "rewards/margins": 0.003621369134634733,
      "rewards/rejected": -0.011593475937843323,
      "step": 9
    },
    {
      "debug/policy_chosen_logits": 22.315250396728516,
      "debug/policy_chosen_logps": -425.8302001953125,
      "debug/policy_rejected_logits": 28.707416534423828,
      "debug/policy_rejected_logps": -453.5276184082031,
      "debug/reference_chosen_logps": -427.7785339355469,
      "debug/reference_rejected_logps": -450.7994079589844,
      "epoch": 0.10989010989010989,
      "grad_norm": 6.765369287722621,
      "learning_rate": 1e-06,
      "logits/chosen": 22.315250396728516,
      "logits/rejected": 28.707416534423828,
      "logps/chosen": -425.8302001953125,
      "logps/rejected": -453.5276184082031,
      "loss": 0.4488,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.019483529031276703,
      "rewards/margins": 0.04676567018032074,
      "rewards/rejected": -0.027282143011689186,
      "step": 10
    },
    {
      "debug/policy_chosen_logits": 23.35513687133789,
      "debug/policy_chosen_logps": -409.0987548828125,
      "debug/policy_rejected_logits": 25.19911003112793,
      "debug/policy_rejected_logps": -457.9080810546875,
      "debug/reference_chosen_logps": -411.0341796875,
      "debug/reference_rejected_logps": -452.00286865234375,
      "epoch": 0.12087912087912088,
      "grad_norm": 6.194229784289876,
      "learning_rate": 1e-06,
      "logits/chosen": 23.35513687133789,
      "logits/rejected": 25.19911003112793,
      "logps/chosen": -409.0987548828125,
      "logps/rejected": -457.9080810546875,
      "loss": 0.4512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.01935398206114769,
      "rewards/margins": 0.07840606570243835,
      "rewards/rejected": -0.059052083641290665,
      "step": 11
    },
    {
      "debug/policy_chosen_logits": 24.74183464050293,
      "debug/policy_chosen_logps": -443.5227355957031,
      "debug/policy_rejected_logits": 23.557279586791992,
      "debug/policy_rejected_logps": -438.05023193359375,
      "debug/reference_chosen_logps": -444.56549072265625,
      "debug/reference_rejected_logps": -433.29443359375,
      "epoch": 0.13186813186813187,
      "grad_norm": 6.295588890883081,
      "learning_rate": 1e-06,
      "logits/chosen": 24.74183464050293,
      "logits/rejected": 23.557279586791992,
      "logps/chosen": -443.5227355957031,
      "logps/rejected": -438.05023193359375,
      "loss": 0.444,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.010427666828036308,
      "rewards/margins": 0.057986069470644,
      "rewards/rejected": -0.04755840077996254,
      "step": 12
    },
    {
      "debug/policy_chosen_logits": 25.26060676574707,
      "debug/policy_chosen_logps": -401.9018249511719,
      "debug/policy_rejected_logits": 25.899250030517578,
      "debug/policy_rejected_logps": -447.271240234375,
      "debug/reference_chosen_logps": -407.1501159667969,
      "debug/reference_rejected_logps": -443.3197326660156,
      "epoch": 0.14285714285714285,
      "grad_norm": 6.651428295339092,
      "learning_rate": 1e-06,
      "logits/chosen": 25.26060676574707,
      "logits/rejected": 25.899250030517578,
      "logps/chosen": -401.9018249511719,
      "logps/rejected": -447.271240234375,
      "loss": 0.4438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05248305946588516,
      "rewards/margins": 0.0919981375336647,
      "rewards/rejected": -0.03951507434248924,
      "step": 13
    },
    {
      "debug/policy_chosen_logits": 27.350528717041016,
      "debug/policy_chosen_logps": -431.6693115234375,
      "debug/policy_rejected_logits": 27.774742126464844,
      "debug/policy_rejected_logps": -437.4266052246094,
      "debug/reference_chosen_logps": -437.1637268066406,
      "debug/reference_rejected_logps": -433.93536376953125,
      "epoch": 0.15384615384615385,
      "grad_norm": 6.44432847940225,
      "learning_rate": 1e-06,
      "logits/chosen": 27.350528717041016,
      "logits/rejected": 27.774742126464844,
      "logps/chosen": -431.6693115234375,
      "logps/rejected": -437.4266052246094,
      "loss": 0.4203,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05494399741292,
      "rewards/margins": 0.08985621482133865,
      "rewards/rejected": -0.034912221133708954,
      "step": 14
    },
    {
      "debug/policy_chosen_logits": 26.464439392089844,
      "debug/policy_chosen_logps": -441.0960998535156,
      "debug/policy_rejected_logits": 22.606828689575195,
      "debug/policy_rejected_logps": -450.34637451171875,
      "debug/reference_chosen_logps": -443.2272033691406,
      "debug/reference_rejected_logps": -445.57098388671875,
      "epoch": 0.16483516483516483,
      "grad_norm": 6.034773405123565,
      "learning_rate": 1e-06,
      "logits/chosen": 26.464439392089844,
      "logits/rejected": 22.606828689575195,
      "logps/chosen": -441.0960998535156,
      "logps/rejected": -450.34637451171875,
      "loss": 0.4508,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.021311109885573387,
      "rewards/margins": 0.06906504929065704,
      "rewards/rejected": -0.047753941267728806,
      "step": 15
    },
    {
      "debug/policy_chosen_logits": 23.063968658447266,
      "debug/policy_chosen_logps": -430.00677490234375,
      "debug/policy_rejected_logits": 30.106121063232422,
      "debug/policy_rejected_logps": -459.87164306640625,
      "debug/reference_chosen_logps": -435.9080810546875,
      "debug/reference_rejected_logps": -454.621337890625,
      "epoch": 0.17582417582417584,
      "grad_norm": 5.694131268125543,
      "learning_rate": 1e-06,
      "logits/chosen": 23.063968658447266,
      "logits/rejected": 30.106121063232422,
      "logps/chosen": -430.00677490234375,
      "logps/rejected": -459.87164306640625,
      "loss": 0.4059,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05901290848851204,
      "rewards/margins": 0.11151611059904099,
      "rewards/rejected": -0.05250319838523865,
      "step": 16
    },
    {
      "debug/policy_chosen_logits": 29.839580535888672,
      "debug/policy_chosen_logps": -436.97039794921875,
      "debug/policy_rejected_logits": 28.98643684387207,
      "debug/policy_rejected_logps": -476.59881591796875,
      "debug/reference_chosen_logps": -443.8123779296875,
      "debug/reference_rejected_logps": -466.94873046875,
      "epoch": 0.18681318681318682,
      "grad_norm": 6.738937580645961,
      "learning_rate": 1e-06,
      "logits/chosen": 29.839580535888672,
      "logits/rejected": 28.98643684387207,
      "logps/chosen": -436.97039794921875,
      "logps/rejected": -476.59881591796875,
      "loss": 0.4118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06841972470283508,
      "rewards/margins": 0.16492080688476562,
      "rewards/rejected": -0.09650108218193054,
      "step": 17
    },
    {
      "debug/policy_chosen_logits": 29.84876823425293,
      "debug/policy_chosen_logps": -441.1398620605469,
      "debug/policy_rejected_logits": 21.603878021240234,
      "debug/policy_rejected_logps": -443.14556884765625,
      "debug/reference_chosen_logps": -448.20770263671875,
      "debug/reference_rejected_logps": -441.1372375488281,
      "epoch": 0.1978021978021978,
      "grad_norm": 5.137388346436403,
      "learning_rate": 1e-06,
      "logits/chosen": 29.84876823425293,
      "logits/rejected": 21.603878021240234,
      "logps/chosen": -441.1398620605469,
      "logps/rejected": -443.14556884765625,
      "loss": 0.4207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07067855447530746,
      "rewards/margins": 0.09076160192489624,
      "rewards/rejected": -0.020083043724298477,
      "step": 18
    },
    {
      "debug/policy_chosen_logits": 26.974803924560547,
      "debug/policy_chosen_logps": -453.483642578125,
      "debug/policy_rejected_logits": 26.663101196289062,
      "debug/policy_rejected_logps": -489.6366271972656,
      "debug/reference_chosen_logps": -460.9517822265625,
      "debug/reference_rejected_logps": -470.0023193359375,
      "epoch": 0.2087912087912088,
      "grad_norm": 7.17158785388267,
      "learning_rate": 1e-06,
      "logits/chosen": 26.974803924560547,
      "logits/rejected": 26.663101196289062,
      "logps/chosen": -453.483642578125,
      "logps/rejected": -489.6366271972656,
      "loss": 0.3364,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07468143105506897,
      "rewards/margins": 0.271024614572525,
      "rewards/rejected": -0.19634319841861725,
      "step": 19
    },
    {
      "debug/policy_chosen_logits": 25.026630401611328,
      "debug/policy_chosen_logps": -433.65960693359375,
      "debug/policy_rejected_logits": 25.77982521057129,
      "debug/policy_rejected_logps": -471.0029296875,
      "debug/reference_chosen_logps": -436.43359375,
      "debug/reference_rejected_logps": -460.9761962890625,
      "epoch": 0.21978021978021978,
      "grad_norm": 4.898911355829737,
      "learning_rate": 1e-06,
      "logits/chosen": 25.026630401611328,
      "logits/rejected": 25.77982521057129,
      "logps/chosen": -433.65960693359375,
      "logps/rejected": -471.0029296875,
      "loss": 0.3395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.027739638462662697,
      "rewards/margins": 0.12800678610801697,
      "rewards/rejected": -0.10026714205741882,
      "step": 20
    },
    {
      "debug/policy_chosen_logits": 25.650062561035156,
      "debug/policy_chosen_logps": -435.83905029296875,
      "debug/policy_rejected_logits": 24.800756454467773,
      "debug/policy_rejected_logps": -481.5117492675781,
      "debug/reference_chosen_logps": -443.297607421875,
      "debug/reference_rejected_logps": -455.4820556640625,
      "epoch": 0.23076923076923078,
      "grad_norm": 5.689326640738984,
      "learning_rate": 1e-06,
      "logits/chosen": 25.650062561035156,
      "logits/rejected": 24.800756454467773,
      "logps/chosen": -435.83905029296875,
      "logps/rejected": -481.5117492675781,
      "loss": 0.3623,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07458514720201492,
      "rewards/margins": 0.3348817229270935,
      "rewards/rejected": -0.2602965533733368,
      "step": 21
    },
    {
      "debug/policy_chosen_logits": 22.519716262817383,
      "debug/policy_chosen_logps": -405.88226318359375,
      "debug/policy_rejected_logits": 27.31670570373535,
      "debug/policy_rejected_logps": -470.7935791015625,
      "debug/reference_chosen_logps": -416.4732360839844,
      "debug/reference_rejected_logps": -464.93170166015625,
      "epoch": 0.24175824175824176,
      "grad_norm": 4.9841849550976765,
      "learning_rate": 1e-06,
      "logits/chosen": 22.519716262817383,
      "logits/rejected": 27.31670570373535,
      "logps/chosen": -405.88226318359375,
      "logps/rejected": -470.7935791015625,
      "loss": 0.3153,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10590964555740356,
      "rewards/margins": 0.16452865302562714,
      "rewards/rejected": -0.058619000017642975,
      "step": 22
    },
    {
      "debug/policy_chosen_logits": 25.54558753967285,
      "debug/policy_chosen_logps": -427.68450927734375,
      "debug/policy_rejected_logits": 28.938701629638672,
      "debug/policy_rejected_logps": -456.0726623535156,
      "debug/reference_chosen_logps": -445.5054626464844,
      "debug/reference_rejected_logps": -441.0499267578125,
      "epoch": 0.25274725274725274,
      "grad_norm": 4.453741538704258,
      "learning_rate": 1e-06,
      "logits/chosen": 25.54558753967285,
      "logits/rejected": 28.938701629638672,
      "logps/chosen": -427.68450927734375,
      "logps/rejected": -456.0726623535156,
      "loss": 0.3437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17820952832698822,
      "rewards/margins": 0.328436940908432,
      "rewards/rejected": -0.1502273976802826,
      "step": 23
    },
    {
      "debug/policy_chosen_logits": 24.3718318939209,
      "debug/policy_chosen_logps": -407.55877685546875,
      "debug/policy_rejected_logits": 25.362825393676758,
      "debug/policy_rejected_logps": -428.3117370605469,
      "debug/reference_chosen_logps": -422.75518798828125,
      "debug/reference_rejected_logps": -419.3153381347656,
      "epoch": 0.26373626373626374,
      "grad_norm": 5.345678599763925,
      "learning_rate": 1e-06,
      "logits/chosen": 24.3718318939209,
      "logits/rejected": 25.362825393676758,
      "logps/chosen": -407.55877685546875,
      "logps/rejected": -428.3117370605469,
      "loss": 0.3313,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1519637256860733,
      "rewards/margins": 0.24192774295806885,
      "rewards/rejected": -0.08996403217315674,
      "step": 24
    },
    {
      "debug/policy_chosen_logits": 22.36195945739746,
      "debug/policy_chosen_logps": -407.60107421875,
      "debug/policy_rejected_logits": 21.827068328857422,
      "debug/policy_rejected_logps": -436.4155578613281,
      "debug/reference_chosen_logps": -428.3585205078125,
      "debug/reference_rejected_logps": -423.5442199707031,
      "epoch": 0.27472527472527475,
      "grad_norm": 6.983328118149161,
      "learning_rate": 1e-06,
      "logits/chosen": 22.36195945739746,
      "logits/rejected": 21.827068328857422,
      "logps/chosen": -407.60107421875,
      "logps/rejected": -436.4155578613281,
      "loss": 0.3456,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20757438242435455,
      "rewards/margins": 0.3362876772880554,
      "rewards/rejected": -0.12871329486370087,
      "step": 25
    },
    {
      "debug/policy_chosen_logits": 24.60111427307129,
      "debug/policy_chosen_logps": -424.568115234375,
      "debug/policy_rejected_logits": 27.44658088684082,
      "debug/policy_rejected_logps": -500.7226867675781,
      "debug/reference_chosen_logps": -442.6761779785156,
      "debug/reference_rejected_logps": -472.3526916503906,
      "epoch": 0.2857142857142857,
      "grad_norm": 4.621624916905314,
      "learning_rate": 1e-06,
      "logits/chosen": 24.60111427307129,
      "logits/rejected": 27.44658088684082,
      "logps/chosen": -424.568115234375,
      "logps/rejected": -500.7226867675781,
      "loss": 0.3364,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18108057975769043,
      "rewards/margins": 0.46478039026260376,
      "rewards/rejected": -0.28369981050491333,
      "step": 26
    },
    {
      "debug/policy_chosen_logits": 24.04281234741211,
      "debug/policy_chosen_logps": -420.4295349121094,
      "debug/policy_rejected_logits": 24.854686737060547,
      "debug/policy_rejected_logps": -453.5174560546875,
      "debug/reference_chosen_logps": -440.0169677734375,
      "debug/reference_rejected_logps": -441.5909118652344,
      "epoch": 0.2967032967032967,
      "grad_norm": 4.722287417720129,
      "learning_rate": 1e-06,
      "logits/chosen": 24.04281234741211,
      "logits/rejected": 24.854686737060547,
      "logps/chosen": -420.4295349121094,
      "logps/rejected": -453.5174560546875,
      "loss": 0.3288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19587397575378418,
      "rewards/margins": 0.3151397109031677,
      "rewards/rejected": -0.11926574259996414,
      "step": 27
    },
    {
      "debug/policy_chosen_logits": 26.117721557617188,
      "debug/policy_chosen_logps": -437.7601318359375,
      "debug/policy_rejected_logits": 28.745031356811523,
      "debug/policy_rejected_logps": -477.95904541015625,
      "debug/reference_chosen_logps": -448.17889404296875,
      "debug/reference_rejected_logps": -449.2839050292969,
      "epoch": 0.3076923076923077,
      "grad_norm": 4.51629611753504,
      "learning_rate": 1e-06,
      "logits/chosen": 26.117721557617188,
      "logits/rejected": 28.745031356811523,
      "logps/chosen": -437.7601318359375,
      "logps/rejected": -477.95904541015625,
      "loss": 0.3783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10418765991926193,
      "rewards/margins": 0.3909391462802887,
      "rewards/rejected": -0.28675150871276855,
      "step": 28
    },
    {
      "debug/policy_chosen_logits": 28.643272399902344,
      "debug/policy_chosen_logps": -445.78875732421875,
      "debug/policy_rejected_logits": 30.509065628051758,
      "debug/policy_rejected_logps": -490.1319274902344,
      "debug/reference_chosen_logps": -458.77532958984375,
      "debug/reference_rejected_logps": -467.02667236328125,
      "epoch": 0.31868131868131866,
      "grad_norm": 5.726900857769083,
      "learning_rate": 1e-06,
      "logits/chosen": 28.643272399902344,
      "logits/rejected": 30.509065628051758,
      "logps/chosen": -445.78875732421875,
      "logps/rejected": -490.1319274902344,
      "loss": 0.3291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1298656463623047,
      "rewards/margins": 0.3609181046485901,
      "rewards/rejected": -0.2310524582862854,
      "step": 29
    },
    {
      "debug/policy_chosen_logits": 23.9057674407959,
      "debug/policy_chosen_logps": -429.0777587890625,
      "debug/policy_rejected_logits": 27.041988372802734,
      "debug/policy_rejected_logps": -489.99102783203125,
      "debug/reference_chosen_logps": -435.3116149902344,
      "debug/reference_rejected_logps": -468.8659973144531,
      "epoch": 0.32967032967032966,
      "grad_norm": 5.589091335127157,
      "learning_rate": 1e-06,
      "logits/chosen": 23.9057674407959,
      "logits/rejected": 27.041988372802734,
      "logps/chosen": -429.0777587890625,
      "logps/rejected": -489.99102783203125,
      "loss": 0.3149,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06233878806233406,
      "rewards/margins": 0.2735891342163086,
      "rewards/rejected": -0.21125034987926483,
      "step": 30
    },
    {
      "debug/policy_chosen_logits": 20.52339744567871,
      "debug/policy_chosen_logps": -417.83367919921875,
      "debug/policy_rejected_logits": 25.475711822509766,
      "debug/policy_rejected_logps": -477.3974914550781,
      "debug/reference_chosen_logps": -429.0983581542969,
      "debug/reference_rejected_logps": -454.0728759765625,
      "epoch": 0.34065934065934067,
      "grad_norm": 5.919614811999328,
      "learning_rate": 1e-06,
      "logits/chosen": 20.52339744567871,
      "logits/rejected": 25.475711822509766,
      "logps/chosen": -417.83367919921875,
      "logps/rejected": -477.3974914550781,
      "loss": 0.2959,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11264674365520477,
      "rewards/margins": 0.34589269757270813,
      "rewards/rejected": -0.23324593901634216,
      "step": 31
    },
    {
      "debug/policy_chosen_logits": 21.121295928955078,
      "debug/policy_chosen_logps": -410.53643798828125,
      "debug/policy_rejected_logits": 27.618637084960938,
      "debug/policy_rejected_logps": -471.409912109375,
      "debug/reference_chosen_logps": -427.6445007324219,
      "debug/reference_rejected_logps": -452.9000244140625,
      "epoch": 0.3516483516483517,
      "grad_norm": 6.235065911009125,
      "learning_rate": 1e-06,
      "logits/chosen": 21.121295928955078,
      "logits/rejected": 27.618637084960938,
      "logps/chosen": -410.53643798828125,
      "logps/rejected": -471.409912109375,
      "loss": 0.3322,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1710808128118515,
      "rewards/margins": 0.35617977380752563,
      "rewards/rejected": -0.18509894609451294,
      "step": 32
    },
    {
      "debug/policy_chosen_logits": 25.020137786865234,
      "debug/policy_chosen_logps": -421.2449951171875,
      "debug/policy_rejected_logits": 26.06194305419922,
      "debug/policy_rejected_logps": -455.33154296875,
      "debug/reference_chosen_logps": -445.73223876953125,
      "debug/reference_rejected_logps": -430.2513427734375,
      "epoch": 0.3626373626373626,
      "grad_norm": 4.220132361607984,
      "learning_rate": 1e-06,
      "logits/chosen": 25.020137786865234,
      "logits/rejected": 26.06194305419922,
      "logps/chosen": -421.2449951171875,
      "logps/rejected": -455.33154296875,
      "loss": 0.2825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24487201869487762,
      "rewards/margins": 0.4956740140914917,
      "rewards/rejected": -0.25080201029777527,
      "step": 33
    },
    {
      "debug/policy_chosen_logits": 28.49104881286621,
      "debug/policy_chosen_logps": -432.42578125,
      "debug/policy_rejected_logits": 26.045143127441406,
      "debug/policy_rejected_logps": -500.81494140625,
      "debug/reference_chosen_logps": -455.1983642578125,
      "debug/reference_rejected_logps": -481.32403564453125,
      "epoch": 0.37362637362637363,
      "grad_norm": 4.102855456579325,
      "learning_rate": 1e-06,
      "logits/chosen": 28.49104881286621,
      "logits/rejected": 26.045143127441406,
      "logps/chosen": -432.42578125,
      "logps/rejected": -500.81494140625,
      "loss": 0.2778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22772569954395294,
      "rewards/margins": 0.422635018825531,
      "rewards/rejected": -0.19490931928157806,
      "step": 34
    },
    {
      "debug/policy_chosen_logits": 23.481491088867188,
      "debug/policy_chosen_logps": -410.7614440917969,
      "debug/policy_rejected_logits": 26.54258155822754,
      "debug/policy_rejected_logps": -461.57659912109375,
      "debug/reference_chosen_logps": -431.06304931640625,
      "debug/reference_rejected_logps": -428.4720458984375,
      "epoch": 0.38461538461538464,
      "grad_norm": 3.930918282446488,
      "learning_rate": 1e-06,
      "logits/chosen": 23.481491088867188,
      "logits/rejected": 26.54258155822754,
      "logps/chosen": -410.7614440917969,
      "logps/rejected": -461.57659912109375,
      "loss": 0.2688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20301616191864014,
      "rewards/margins": 0.5340613126754761,
      "rewards/rejected": -0.33104515075683594,
      "step": 35
    },
    {
      "debug/policy_chosen_logits": 24.361486434936523,
      "debug/policy_chosen_logps": -412.67987060546875,
      "debug/policy_rejected_logits": 27.827251434326172,
      "debug/policy_rejected_logps": -452.28857421875,
      "debug/reference_chosen_logps": -430.6239318847656,
      "debug/reference_rejected_logps": -440.8873596191406,
      "epoch": 0.3956043956043956,
      "grad_norm": 5.7094820058187645,
      "learning_rate": 1e-06,
      "logits/chosen": 24.361486434936523,
      "logits/rejected": 27.827251434326172,
      "logps/chosen": -412.67987060546875,
      "logps/rejected": -452.28857421875,
      "loss": 0.3088,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1794406771659851,
      "rewards/margins": 0.29345306754112244,
      "rewards/rejected": -0.11401237547397614,
      "step": 36
    },
    {
      "debug/policy_chosen_logits": 26.789180755615234,
      "debug/policy_chosen_logps": -419.154296875,
      "debug/policy_rejected_logits": 24.381994247436523,
      "debug/policy_rejected_logps": -443.3766174316406,
      "debug/reference_chosen_logps": -440.4947509765625,
      "debug/reference_rejected_logps": -457.65887451171875,
      "epoch": 0.4065934065934066,
      "grad_norm": 3.962286836465319,
      "learning_rate": 1e-06,
      "logits/chosen": 26.789180755615234,
      "logits/rejected": 24.381994247436523,
      "logps/chosen": -419.154296875,
      "logps/rejected": -443.3766174316406,
      "loss": 0.2823,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.21340444684028625,
      "rewards/margins": 0.07058170437812805,
      "rewards/rejected": 0.1428227573633194,
      "step": 37
    },
    {
      "debug/policy_chosen_logits": 24.922195434570312,
      "debug/policy_chosen_logps": -399.93096923828125,
      "debug/policy_rejected_logits": 27.783912658691406,
      "debug/policy_rejected_logps": -472.2608947753906,
      "debug/reference_chosen_logps": -427.5108642578125,
      "debug/reference_rejected_logps": -462.4599914550781,
      "epoch": 0.4175824175824176,
      "grad_norm": 4.150324748851986,
      "learning_rate": 1e-06,
      "logits/chosen": 24.922195434570312,
      "logits/rejected": 27.783912658691406,
      "logps/chosen": -399.93096923828125,
      "logps/rejected": -472.2608947753906,
      "loss": 0.2398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.27579906582832336,
      "rewards/margins": 0.37380802631378174,
      "rewards/rejected": -0.09800895303487778,
      "step": 38
    },
    {
      "debug/policy_chosen_logits": 22.96780014038086,
      "debug/policy_chosen_logps": -420.74261474609375,
      "debug/policy_rejected_logits": 23.919336318969727,
      "debug/policy_rejected_logps": -439.653076171875,
      "debug/reference_chosen_logps": -433.999755859375,
      "debug/reference_rejected_logps": -421.92962646484375,
      "epoch": 0.42857142857142855,
      "grad_norm": 4.365666660792002,
      "learning_rate": 1e-06,
      "logits/chosen": 22.96780014038086,
      "logits/rejected": 23.919336318969727,
      "logps/chosen": -420.74261474609375,
      "logps/rejected": -439.653076171875,
      "loss": 0.2833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13257166743278503,
      "rewards/margins": 0.3098059296607971,
      "rewards/rejected": -0.17723426222801208,
      "step": 39
    },
    {
      "debug/policy_chosen_logits": 27.75818634033203,
      "debug/policy_chosen_logps": -455.0806884765625,
      "debug/policy_rejected_logits": 27.270263671875,
      "debug/policy_rejected_logps": -468.50927734375,
      "debug/reference_chosen_logps": -462.73138427734375,
      "debug/reference_rejected_logps": -473.1136474609375,
      "epoch": 0.43956043956043955,
      "grad_norm": 4.570887002489462,
      "learning_rate": 1e-06,
      "logits/chosen": 27.75818634033203,
      "logits/rejected": 27.270263671875,
      "logps/chosen": -455.0806884765625,
      "logps/rejected": -468.50927734375,
      "loss": 0.3436,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.07650664448738098,
      "rewards/margins": 0.030462883412837982,
      "rewards/rejected": 0.04604377597570419,
      "step": 40
    },
    {
      "debug/policy_chosen_logits": 25.347915649414062,
      "debug/policy_chosen_logps": -406.87591552734375,
      "debug/policy_rejected_logits": 28.548446655273438,
      "debug/policy_rejected_logps": -464.4383544921875,
      "debug/reference_chosen_logps": -433.459716796875,
      "debug/reference_rejected_logps": -448.8690185546875,
      "epoch": 0.45054945054945056,
      "grad_norm": 3.677831065051595,
      "learning_rate": 1e-06,
      "logits/chosen": 25.347915649414062,
      "logits/rejected": 28.548446655273438,
      "logps/chosen": -406.87591552734375,
      "logps/rejected": -464.4383544921875,
      "loss": 0.2758,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.26583802700042725,
      "rewards/margins": 0.4215315878391266,
      "rewards/rejected": -0.15569356083869934,
      "step": 41
    },
    {
      "debug/policy_chosen_logits": 22.77235221862793,
      "debug/policy_chosen_logps": -402.58416748046875,
      "debug/policy_rejected_logits": 26.393054962158203,
      "debug/policy_rejected_logps": -471.61260986328125,
      "debug/reference_chosen_logps": -426.26690673828125,
      "debug/reference_rejected_logps": -453.03125,
      "epoch": 0.46153846153846156,
      "grad_norm": 3.808742477450644,
      "learning_rate": 1e-06,
      "logits/chosen": 22.77235221862793,
      "logits/rejected": 26.393054962158203,
      "logps/chosen": -402.58416748046875,
      "logps/rejected": -471.61260986328125,
      "loss": 0.2739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.23682719469070435,
      "rewards/margins": 0.4226408302783966,
      "rewards/rejected": -0.18581363558769226,
      "step": 42
    },
    {
      "debug/policy_chosen_logits": 22.165273666381836,
      "debug/policy_chosen_logps": -409.06304931640625,
      "debug/policy_rejected_logits": 26.427730560302734,
      "debug/policy_rejected_logps": -448.95489501953125,
      "debug/reference_chosen_logps": -434.5325927734375,
      "debug/reference_rejected_logps": -412.31060791015625,
      "epoch": 0.4725274725274725,
      "grad_norm": 7.512415554484962,
      "learning_rate": 1e-06,
      "logits/chosen": 22.165273666381836,
      "logits/rejected": 26.427730560302734,
      "logps/chosen": -409.06304931640625,
      "logps/rejected": -448.95489501953125,
      "loss": 0.3186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25469571352005005,
      "rewards/margins": 0.6211386322975159,
      "rewards/rejected": -0.36644288897514343,
      "step": 43
    },
    {
      "debug/policy_chosen_logits": 25.82545280456543,
      "debug/policy_chosen_logps": -409.12420654296875,
      "debug/policy_rejected_logits": 23.266063690185547,
      "debug/policy_rejected_logps": -493.0911865234375,
      "debug/reference_chosen_logps": -437.0023193359375,
      "debug/reference_rejected_logps": -456.59796142578125,
      "epoch": 0.4835164835164835,
      "grad_norm": 5.005310078975177,
      "learning_rate": 1e-06,
      "logits/chosen": 25.82545280456543,
      "logits/rejected": 23.266063690185547,
      "logps/chosen": -409.12420654296875,
      "logps/rejected": -493.0911865234375,
      "loss": 0.2517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.278780996799469,
      "rewards/margins": 0.6437131762504578,
      "rewards/rejected": -0.36493217945098877,
      "step": 44
    },
    {
      "debug/policy_chosen_logits": 28.779375076293945,
      "debug/policy_chosen_logps": -419.06585693359375,
      "debug/policy_rejected_logits": 25.911027908325195,
      "debug/policy_rejected_logps": -493.4896240234375,
      "debug/reference_chosen_logps": -444.6504211425781,
      "debug/reference_rejected_logps": -467.607177734375,
      "epoch": 0.4945054945054945,
      "grad_norm": 5.276153180435082,
      "learning_rate": 1e-06,
      "logits/chosen": 28.779375076293945,
      "logits/rejected": 25.911027908325195,
      "logps/chosen": -419.06585693359375,
      "logps/rejected": -493.4896240234375,
      "loss": 0.3159,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2558458745479584,
      "rewards/margins": 0.5146701335906982,
      "rewards/rejected": -0.25882428884506226,
      "step": 45
    },
    {
      "debug/policy_chosen_logits": 27.05695152282715,
      "debug/policy_chosen_logps": -428.0582275390625,
      "debug/policy_rejected_logits": 27.517444610595703,
      "debug/policy_rejected_logps": -499.21636962890625,
      "debug/reference_chosen_logps": -452.95599365234375,
      "debug/reference_rejected_logps": -468.61138916015625,
      "epoch": 0.5054945054945055,
      "grad_norm": 4.904417802927379,
      "learning_rate": 1e-06,
      "logits/chosen": 27.05695152282715,
      "logits/rejected": 27.517444610595703,
      "logps/chosen": -428.0582275390625,
      "logps/rejected": -499.21636962890625,
      "loss": 0.242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2489778846502304,
      "rewards/margins": 0.5550275444984436,
      "rewards/rejected": -0.306049644947052,
      "step": 46
    },
    {
      "debug/policy_chosen_logits": 24.79572105407715,
      "debug/policy_chosen_logps": -412.3216552734375,
      "debug/policy_rejected_logits": 26.50978660583496,
      "debug/policy_rejected_logps": -466.15557861328125,
      "debug/reference_chosen_logps": -430.951416015625,
      "debug/reference_rejected_logps": -448.00433349609375,
      "epoch": 0.5164835164835165,
      "grad_norm": 5.334062642354243,
      "learning_rate": 1e-06,
      "logits/chosen": 24.79572105407715,
      "logits/rejected": 26.50978660583496,
      "logps/chosen": -412.3216552734375,
      "logps/rejected": -466.15557861328125,
      "loss": 0.3266,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.18629732728004456,
      "rewards/margins": 0.3678096830844879,
      "rewards/rejected": -0.18151238560676575,
      "step": 47
    },
    {
      "debug/policy_chosen_logits": 24.24400520324707,
      "debug/policy_chosen_logps": -392.45751953125,
      "debug/policy_rejected_logits": 27.10297393798828,
      "debug/policy_rejected_logps": -463.1430969238281,
      "debug/reference_chosen_logps": -417.45147705078125,
      "debug/reference_rejected_logps": -438.20220947265625,
      "epoch": 0.5274725274725275,
      "grad_norm": 7.066668036628892,
      "learning_rate": 1e-06,
      "logits/chosen": 24.24400520324707,
      "logits/rejected": 27.10297393798828,
      "logps/chosen": -392.45751953125,
      "logps/rejected": -463.1430969238281,
      "loss": 0.3255,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24993950128555298,
      "rewards/margins": 0.49934816360473633,
      "rewards/rejected": -0.24940869212150574,
      "step": 48
    },
    {
      "debug/policy_chosen_logits": 22.664649963378906,
      "debug/policy_chosen_logps": -394.15032958984375,
      "debug/policy_rejected_logits": 25.70164680480957,
      "debug/policy_rejected_logps": -452.9449768066406,
      "debug/reference_chosen_logps": -416.6987609863281,
      "debug/reference_rejected_logps": -439.2945251464844,
      "epoch": 0.5384615384615384,
      "grad_norm": 5.135410370061012,
      "learning_rate": 1e-06,
      "logits/chosen": 22.664649963378906,
      "logits/rejected": 25.70164680480957,
      "logps/chosen": -394.15032958984375,
      "logps/rejected": -452.9449768066406,
      "loss": 0.2879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22548410296440125,
      "rewards/margins": 0.3619886040687561,
      "rewards/rejected": -0.13650450110435486,
      "step": 49
    },
    {
      "debug/policy_chosen_logits": 22.666519165039062,
      "debug/policy_chosen_logps": -396.6944885253906,
      "debug/policy_rejected_logits": 21.13740348815918,
      "debug/policy_rejected_logps": -453.7715148925781,
      "debug/reference_chosen_logps": -424.0322265625,
      "debug/reference_rejected_logps": -440.89862060546875,
      "epoch": 0.5494505494505495,
      "grad_norm": 3.9756803394109825,
      "learning_rate": 1e-06,
      "logits/chosen": 22.666519165039062,
      "logits/rejected": 21.13740348815918,
      "logps/chosen": -396.6944885253906,
      "logps/rejected": -453.7715148925781,
      "loss": 0.2293,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.27337703108787537,
      "rewards/margins": 0.40210601687431335,
      "rewards/rejected": -0.1287289708852768,
      "step": 50
    },
    {
      "debug/policy_chosen_logits": 23.19964027404785,
      "debug/policy_chosen_logps": -397.15765380859375,
      "debug/policy_rejected_logits": 24.41975212097168,
      "debug/policy_rejected_logps": -475.76104736328125,
      "debug/reference_chosen_logps": -424.8116760253906,
      "debug/reference_rejected_logps": -434.97528076171875,
      "epoch": 0.5604395604395604,
      "grad_norm": 4.600442889147922,
      "learning_rate": 1e-06,
      "logits/chosen": 23.19964027404785,
      "logits/rejected": 24.41975212097168,
      "logps/chosen": -397.15765380859375,
      "logps/rejected": -475.76104736328125,
      "loss": 0.2753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2765401601791382,
      "rewards/margins": 0.6843976974487305,
      "rewards/rejected": -0.4078575372695923,
      "step": 51
    },
    {
      "debug/policy_chosen_logits": 27.027416229248047,
      "debug/policy_chosen_logps": -433.012451171875,
      "debug/policy_rejected_logits": 24.729970932006836,
      "debug/policy_rejected_logps": -465.9927978515625,
      "debug/reference_chosen_logps": -459.7962951660156,
      "debug/reference_rejected_logps": -439.83599853515625,
      "epoch": 0.5714285714285714,
      "grad_norm": 3.8308230544471087,
      "learning_rate": 1e-06,
      "logits/chosen": 27.027416229248047,
      "logits/rejected": 24.729970932006836,
      "logps/chosen": -433.012451171875,
      "logps/rejected": -465.9927978515625,
      "loss": 0.2846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26783838868141174,
      "rewards/margins": 0.529405951499939,
      "rewards/rejected": -0.26156753301620483,
      "step": 52
    },
    {
      "debug/policy_chosen_logits": 25.013721466064453,
      "debug/policy_chosen_logps": -409.18426513671875,
      "debug/policy_rejected_logits": 29.04503059387207,
      "debug/policy_rejected_logps": -455.4191589355469,
      "debug/reference_chosen_logps": -434.70855712890625,
      "debug/reference_rejected_logps": -437.6434020996094,
      "epoch": 0.5824175824175825,
      "grad_norm": 5.361663918449101,
      "learning_rate": 1e-06,
      "logits/chosen": 25.013721466064453,
      "logits/rejected": 29.04503059387207,
      "logps/chosen": -409.18426513671875,
      "logps/rejected": -455.4191589355469,
      "loss": 0.3145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2552429437637329,
      "rewards/margins": 0.433000773191452,
      "rewards/rejected": -0.17775781452655792,
      "step": 53
    },
    {
      "debug/policy_chosen_logits": 26.417736053466797,
      "debug/policy_chosen_logps": -419.6263427734375,
      "debug/policy_rejected_logits": 28.78829002380371,
      "debug/policy_rejected_logps": -475.8312683105469,
      "debug/reference_chosen_logps": -440.78143310546875,
      "debug/reference_rejected_logps": -462.00067138671875,
      "epoch": 0.5934065934065934,
      "grad_norm": 4.416192021518405,
      "learning_rate": 1e-06,
      "logits/chosen": 26.417736053466797,
      "logits/rejected": 28.78829002380371,
      "logps/chosen": -419.6263427734375,
      "logps/rejected": -475.8312683105469,
      "loss": 0.3234,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2115507870912552,
      "rewards/margins": 0.34985676407814026,
      "rewards/rejected": -0.13830597698688507,
      "step": 54
    },
    {
      "debug/policy_chosen_logits": 28.22123908996582,
      "debug/policy_chosen_logps": -429.47418212890625,
      "debug/policy_rejected_logits": 25.79034996032715,
      "debug/policy_rejected_logps": -471.1182861328125,
      "debug/reference_chosen_logps": -445.8591003417969,
      "debug/reference_rejected_logps": -455.14202880859375,
      "epoch": 0.6043956043956044,
      "grad_norm": 6.671276859718192,
      "learning_rate": 1e-06,
      "logits/chosen": 28.22123908996582,
      "logits/rejected": 25.79034996032715,
      "logps/chosen": -429.47418212890625,
      "logps/rejected": -471.1182861328125,
      "loss": 0.2846,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16384944319725037,
      "rewards/margins": 0.32361170649528503,
      "rewards/rejected": -0.15976226329803467,
      "step": 55
    },
    {
      "debug/policy_chosen_logits": 26.506807327270508,
      "debug/policy_chosen_logps": -420.41082763671875,
      "debug/policy_rejected_logits": 27.579261779785156,
      "debug/policy_rejected_logps": -452.91162109375,
      "debug/reference_chosen_logps": -448.4418029785156,
      "debug/reference_rejected_logps": -432.2170104980469,
      "epoch": 0.6153846153846154,
      "grad_norm": 5.601125759934195,
      "learning_rate": 1e-06,
      "logits/chosen": 26.506807327270508,
      "logits/rejected": 27.579261779785156,
      "logps/chosen": -420.41082763671875,
      "logps/rejected": -452.91162109375,
      "loss": 0.2563,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2803099453449249,
      "rewards/margins": 0.48725610971450806,
      "rewards/rejected": -0.20694613456726074,
      "step": 56
    },
    {
      "debug/policy_chosen_logits": 22.477113723754883,
      "debug/policy_chosen_logps": -410.53009033203125,
      "debug/policy_rejected_logits": 24.827415466308594,
      "debug/policy_rejected_logps": -463.63214111328125,
      "debug/reference_chosen_logps": -430.95086669921875,
      "debug/reference_rejected_logps": -428.8712158203125,
      "epoch": 0.6263736263736264,
      "grad_norm": 4.278326605774298,
      "learning_rate": 1e-06,
      "logits/chosen": 22.477113723754883,
      "logits/rejected": 24.827415466308594,
      "logps/chosen": -410.53009033203125,
      "logps/rejected": -463.63214111328125,
      "loss": 0.2727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20420792698860168,
      "rewards/margins": 0.5518174171447754,
      "rewards/rejected": -0.3476094603538513,
      "step": 57
    },
    {
      "debug/policy_chosen_logits": 24.6320858001709,
      "debug/policy_chosen_logps": -408.2926025390625,
      "debug/policy_rejected_logits": 27.864049911499023,
      "debug/policy_rejected_logps": -461.57208251953125,
      "debug/reference_chosen_logps": -426.1760559082031,
      "debug/reference_rejected_logps": -447.0111083984375,
      "epoch": 0.6373626373626373,
      "grad_norm": 4.643648875549545,
      "learning_rate": 1e-06,
      "logits/chosen": 24.6320858001709,
      "logits/rejected": 27.864049911499023,
      "logps/chosen": -408.2926025390625,
      "logps/rejected": -461.57208251953125,
      "loss": 0.2516,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.17883440852165222,
      "rewards/margins": 0.3244439363479614,
      "rewards/rejected": -0.1456095427274704,
      "step": 58
    },
    {
      "debug/policy_chosen_logits": 20.885883331298828,
      "debug/policy_chosen_logps": -412.9990539550781,
      "debug/policy_rejected_logits": 21.050661087036133,
      "debug/policy_rejected_logps": -522.6400756835938,
      "debug/reference_chosen_logps": -440.10931396484375,
      "debug/reference_rejected_logps": -487.7521667480469,
      "epoch": 0.6483516483516484,
      "grad_norm": 4.05657059697319,
      "learning_rate": 1e-06,
      "logits/chosen": 20.885883331298828,
      "logits/rejected": 21.050661087036133,
      "logps/chosen": -412.9990539550781,
      "logps/rejected": -522.6400756835938,
      "loss": 0.2678,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2711024880409241,
      "rewards/margins": 0.6199814081192017,
      "rewards/rejected": -0.3488789200782776,
      "step": 59
    },
    {
      "debug/policy_chosen_logits": 24.035409927368164,
      "debug/policy_chosen_logps": -399.28765869140625,
      "debug/policy_rejected_logits": 24.84585952758789,
      "debug/policy_rejected_logps": -453.340576171875,
      "debug/reference_chosen_logps": -420.63153076171875,
      "debug/reference_rejected_logps": -434.2669677734375,
      "epoch": 0.6593406593406593,
      "grad_norm": 3.784225958996139,
      "learning_rate": 1e-06,
      "logits/chosen": 24.035409927368164,
      "logits/rejected": 24.84585952758789,
      "logps/chosen": -399.28765869140625,
      "logps/rejected": -453.340576171875,
      "loss": 0.2552,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2134389877319336,
      "rewards/margins": 0.4041747748851776,
      "rewards/rejected": -0.19073577225208282,
      "step": 60
    },
    {
      "debug/policy_chosen_logits": 21.771154403686523,
      "debug/policy_chosen_logps": -403.99932861328125,
      "debug/policy_rejected_logits": 25.54006576538086,
      "debug/policy_rejected_logps": -431.0410461425781,
      "debug/reference_chosen_logps": -427.13372802734375,
      "debug/reference_rejected_logps": -426.09619140625,
      "epoch": 0.6703296703296703,
      "grad_norm": 4.4444525496723815,
      "learning_rate": 1e-06,
      "logits/chosen": 21.771154403686523,
      "logits/rejected": 25.54006576538086,
      "logps/chosen": -403.99932861328125,
      "logps/rejected": -431.0410461425781,
      "loss": 0.2411,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2313441038131714,
      "rewards/margins": 0.2807927131652832,
      "rewards/rejected": -0.04944861680269241,
      "step": 61
    },
    {
      "debug/policy_chosen_logits": 21.56890869140625,
      "debug/policy_chosen_logps": -418.62939453125,
      "debug/policy_rejected_logits": 26.892208099365234,
      "debug/policy_rejected_logps": -432.6386413574219,
      "debug/reference_chosen_logps": -446.4366149902344,
      "debug/reference_rejected_logps": -437.1725769042969,
      "epoch": 0.6813186813186813,
      "grad_norm": 6.207177159902563,
      "learning_rate": 1e-06,
      "logits/chosen": 21.56890869140625,
      "logits/rejected": 26.892208099365234,
      "logps/chosen": -418.62939453125,
      "logps/rejected": -432.6386413574219,
      "loss": 0.2765,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.27807170152664185,
      "rewards/margins": 0.23273253440856934,
      "rewards/rejected": 0.04533915966749191,
      "step": 62
    },
    {
      "debug/policy_chosen_logits": 28.32245445251465,
      "debug/policy_chosen_logps": -440.50726318359375,
      "debug/policy_rejected_logits": 25.365680694580078,
      "debug/policy_rejected_logps": -464.0654296875,
      "debug/reference_chosen_logps": -475.3460998535156,
      "debug/reference_rejected_logps": -457.081787109375,
      "epoch": 0.6923076923076923,
      "grad_norm": 5.48574215281406,
      "learning_rate": 1e-06,
      "logits/chosen": 28.32245445251465,
      "logits/rejected": 25.365680694580078,
      "logps/chosen": -440.50726318359375,
      "logps/rejected": -464.0654296875,
      "loss": 0.2435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34838855266571045,
      "rewards/margins": 0.4182246923446655,
      "rewards/rejected": -0.06983615458011627,
      "step": 63
    },
    {
      "debug/policy_chosen_logits": 25.090259552001953,
      "debug/policy_chosen_logps": -396.6990051269531,
      "debug/policy_rejected_logits": 27.255340576171875,
      "debug/policy_rejected_logps": -454.5365295410156,
      "debug/reference_chosen_logps": -436.3953552246094,
      "debug/reference_rejected_logps": -430.6047058105469,
      "epoch": 0.7032967032967034,
      "grad_norm": 8.801285897449953,
      "learning_rate": 1e-06,
      "logits/chosen": 25.090259552001953,
      "logits/rejected": 27.255340576171875,
      "logps/chosen": -396.6990051269531,
      "logps/rejected": -454.5365295410156,
      "loss": 0.3064,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3969634771347046,
      "rewards/margins": 0.6362816095352173,
      "rewards/rejected": -0.23931819200515747,
      "step": 64
    },
    {
      "debug/policy_chosen_logits": 28.422582626342773,
      "debug/policy_chosen_logps": -421.810302734375,
      "debug/policy_rejected_logits": 22.969980239868164,
      "debug/policy_rejected_logps": -464.9712219238281,
      "debug/reference_chosen_logps": -445.0280456542969,
      "debug/reference_rejected_logps": -451.85345458984375,
      "epoch": 0.7142857142857143,
      "grad_norm": 3.3417383455535807,
      "learning_rate": 1e-06,
      "logits/chosen": 28.422582626342773,
      "logits/rejected": 22.969980239868164,
      "logps/chosen": -421.810302734375,
      "logps/rejected": -464.9712219238281,
      "loss": 0.216,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2321772277355194,
      "rewards/margins": 0.36335471272468567,
      "rewards/rejected": -0.13117747008800507,
      "step": 65
    },
    {
      "debug/policy_chosen_logits": 24.602392196655273,
      "debug/policy_chosen_logps": -409.503662109375,
      "debug/policy_rejected_logits": 26.957948684692383,
      "debug/policy_rejected_logps": -518.37353515625,
      "debug/reference_chosen_logps": -443.19598388671875,
      "debug/reference_rejected_logps": -475.41815185546875,
      "epoch": 0.7252747252747253,
      "grad_norm": 7.382953809999285,
      "learning_rate": 1e-06,
      "logits/chosen": 24.602392196655273,
      "logits/rejected": 26.957948684692383,
      "logps/chosen": -409.503662109375,
      "logps/rejected": -518.37353515625,
      "loss": 0.1895,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3369232416152954,
      "rewards/margins": 0.7664777040481567,
      "rewards/rejected": -0.42955446243286133,
      "step": 66
    },
    {
      "debug/policy_chosen_logits": 22.386083602905273,
      "debug/policy_chosen_logps": -382.5001220703125,
      "debug/policy_rejected_logits": 16.90252113342285,
      "debug/policy_rejected_logps": -475.36181640625,
      "debug/reference_chosen_logps": -412.29132080078125,
      "debug/reference_rejected_logps": -449.7041015625,
      "epoch": 0.7362637362637363,
      "grad_norm": 5.308279817037437,
      "learning_rate": 1e-06,
      "logits/chosen": 22.386083602905273,
      "logits/rejected": 16.90252113342285,
      "logps/chosen": -382.5001220703125,
      "logps/rejected": -475.36181640625,
      "loss": 0.2588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2979119122028351,
      "rewards/margins": 0.5544889569282532,
      "rewards/rejected": -0.2565770447254181,
      "step": 67
    },
    {
      "debug/policy_chosen_logits": 17.921504974365234,
      "debug/policy_chosen_logps": -390.6407470703125,
      "debug/policy_rejected_logits": 24.367341995239258,
      "debug/policy_rejected_logps": -484.7353515625,
      "debug/reference_chosen_logps": -416.8155822753906,
      "debug/reference_rejected_logps": -466.807861328125,
      "epoch": 0.7472527472527473,
      "grad_norm": 5.3343691933569595,
      "learning_rate": 1e-06,
      "logits/chosen": 17.921504974365234,
      "logits/rejected": 24.367341995239258,
      "logps/chosen": -390.6407470703125,
      "logps/rejected": -484.7353515625,
      "loss": 0.2477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2617482841014862,
      "rewards/margins": 0.441023588180542,
      "rewards/rejected": -0.17927534878253937,
      "step": 68
    },
    {
      "debug/policy_chosen_logits": 22.636571884155273,
      "debug/policy_chosen_logps": -391.5332336425781,
      "debug/policy_rejected_logits": 27.57520294189453,
      "debug/policy_rejected_logps": -491.03448486328125,
      "debug/reference_chosen_logps": -418.2889709472656,
      "debug/reference_rejected_logps": -467.211181640625,
      "epoch": 0.7582417582417582,
      "grad_norm": 4.491987560088097,
      "learning_rate": 1e-06,
      "logits/chosen": 22.636571884155273,
      "logits/rejected": 27.57520294189453,
      "logps/chosen": -391.5332336425781,
      "logps/rejected": -491.03448486328125,
      "loss": 0.2792,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.26755744218826294,
      "rewards/margins": 0.5057904124259949,
      "rewards/rejected": -0.23823297023773193,
      "step": 69
    },
    {
      "debug/policy_chosen_logits": 27.65935516357422,
      "debug/policy_chosen_logps": -403.2398986816406,
      "debug/policy_rejected_logits": 29.533405303955078,
      "debug/policy_rejected_logps": -492.2855224609375,
      "debug/reference_chosen_logps": -433.0833435058594,
      "debug/reference_rejected_logps": -444.8511962890625,
      "epoch": 0.7692307692307693,
      "grad_norm": 3.4461660659523656,
      "learning_rate": 1e-06,
      "logits/chosen": 27.65935516357422,
      "logits/rejected": 29.533405303955078,
      "logps/chosen": -403.2398986816406,
      "logps/rejected": -492.2855224609375,
      "loss": 0.2317,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.29843446612358093,
      "rewards/margins": 0.7727776169776917,
      "rewards/rejected": -0.47434312105178833,
      "step": 70
    },
    {
      "debug/policy_chosen_logits": 24.503950119018555,
      "debug/policy_chosen_logps": -446.45513916015625,
      "debug/policy_rejected_logits": 22.433277130126953,
      "debug/policy_rejected_logps": -470.9190979003906,
      "debug/reference_chosen_logps": -472.0450134277344,
      "debug/reference_rejected_logps": -448.1822814941406,
      "epoch": 0.7802197802197802,
      "grad_norm": 5.717761852601763,
      "learning_rate": 1e-06,
      "logits/chosen": 24.503950119018555,
      "logits/rejected": 22.433277130126953,
      "logps/chosen": -446.45513916015625,
      "logps/rejected": -470.9190979003906,
      "loss": 0.3119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2558988630771637,
      "rewards/margins": 0.4832671284675598,
      "rewards/rejected": -0.22736826539039612,
      "step": 71
    },
    {
      "debug/policy_chosen_logits": 19.840513229370117,
      "debug/policy_chosen_logps": -389.92559814453125,
      "debug/policy_rejected_logits": 24.959850311279297,
      "debug/policy_rejected_logps": -467.8755187988281,
      "debug/reference_chosen_logps": -414.42547607421875,
      "debug/reference_rejected_logps": -441.78948974609375,
      "epoch": 0.7912087912087912,
      "grad_norm": 5.324274056234465,
      "learning_rate": 1e-06,
      "logits/chosen": 19.840513229370117,
      "logits/rejected": 24.959850311279297,
      "logps/chosen": -389.92559814453125,
      "logps/rejected": -467.8755187988281,
      "loss": 0.225,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24499861896038055,
      "rewards/margins": 0.5058590173721313,
      "rewards/rejected": -0.260860413312912,
      "step": 72
    },
    {
      "debug/policy_chosen_logits": 24.089357376098633,
      "debug/policy_chosen_logps": -417.7742614746094,
      "debug/policy_rejected_logits": 26.09589958190918,
      "debug/policy_rejected_logps": -444.3390808105469,
      "debug/reference_chosen_logps": -446.7651672363281,
      "debug/reference_rejected_logps": -417.492919921875,
      "epoch": 0.8021978021978022,
      "grad_norm": 3.7830573167149204,
      "learning_rate": 1e-06,
      "logits/chosen": 24.089357376098633,
      "logits/rejected": 26.09589958190918,
      "logps/chosen": -417.7742614746094,
      "logps/rejected": -444.3390808105469,
      "loss": 0.2977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2899090647697449,
      "rewards/margins": 0.5583702921867371,
      "rewards/rejected": -0.2684612572193146,
      "step": 73
    },
    {
      "debug/policy_chosen_logits": 21.799409866333008,
      "debug/policy_chosen_logps": -405.78070068359375,
      "debug/policy_rejected_logits": 26.489110946655273,
      "debug/policy_rejected_logps": -476.9659423828125,
      "debug/reference_chosen_logps": -435.2866516113281,
      "debug/reference_rejected_logps": -449.07806396484375,
      "epoch": 0.8131868131868132,
      "grad_norm": 3.635552390173908,
      "learning_rate": 1e-06,
      "logits/chosen": 21.799409866333008,
      "logits/rejected": 26.489110946655273,
      "logps/chosen": -405.78070068359375,
      "logps/rejected": -476.9659423828125,
      "loss": 0.2695,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2950596511363983,
      "rewards/margins": 0.573938250541687,
      "rewards/rejected": -0.2788785696029663,
      "step": 74
    },
    {
      "debug/policy_chosen_logits": 21.563961029052734,
      "debug/policy_chosen_logps": -382.45452880859375,
      "debug/policy_rejected_logits": 27.974708557128906,
      "debug/policy_rejected_logps": -446.31573486328125,
      "debug/reference_chosen_logps": -403.13330078125,
      "debug/reference_rejected_logps": -445.6566467285156,
      "epoch": 0.8241758241758241,
      "grad_norm": 6.518551807981199,
      "learning_rate": 1e-06,
      "logits/chosen": 21.563961029052734,
      "logits/rejected": 27.974708557128906,
      "logps/chosen": -382.45452880859375,
      "logps/rejected": -446.31573486328125,
      "loss": 0.339,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2067876011133194,
      "rewards/margins": 0.2133782058954239,
      "rewards/rejected": -0.006590619683265686,
      "step": 75
    },
    {
      "debug/policy_chosen_logits": 26.57479476928711,
      "debug/policy_chosen_logps": -421.49554443359375,
      "debug/policy_rejected_logits": 25.88642692565918,
      "debug/policy_rejected_logps": -463.5325622558594,
      "debug/reference_chosen_logps": -445.17974853515625,
      "debug/reference_rejected_logps": -448.434814453125,
      "epoch": 0.8351648351648352,
      "grad_norm": 4.642048379046934,
      "learning_rate": 1e-06,
      "logits/chosen": 26.57479476928711,
      "logits/rejected": 25.88642692565918,
      "logps/chosen": -421.49554443359375,
      "logps/rejected": -463.5325622558594,
      "loss": 0.3014,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.23684199154376984,
      "rewards/margins": 0.3878192901611328,
      "rewards/rejected": -0.15097728371620178,
      "step": 76
    },
    {
      "debug/policy_chosen_logits": 27.36321258544922,
      "debug/policy_chosen_logps": -417.56976318359375,
      "debug/policy_rejected_logits": 27.49226951599121,
      "debug/policy_rejected_logps": -499.5318603515625,
      "debug/reference_chosen_logps": -444.1575927734375,
      "debug/reference_rejected_logps": -475.24273681640625,
      "epoch": 0.8461538461538461,
      "grad_norm": 4.237463467519612,
      "learning_rate": 1e-06,
      "logits/chosen": 27.36321258544922,
      "logits/rejected": 27.49226951599121,
      "logps/chosen": -417.56976318359375,
      "logps/rejected": -499.5318603515625,
      "loss": 0.2199,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2658780515193939,
      "rewards/margins": 0.5087695121765137,
      "rewards/rejected": -0.24289149045944214,
      "step": 77
    },
    {
      "debug/policy_chosen_logits": 24.683202743530273,
      "debug/policy_chosen_logps": -409.4302673339844,
      "debug/policy_rejected_logits": 23.952590942382812,
      "debug/policy_rejected_logps": -463.2736511230469,
      "debug/reference_chosen_logps": -432.84686279296875,
      "debug/reference_rejected_logps": -437.69964599609375,
      "epoch": 0.8571428571428571,
      "grad_norm": 4.166956738488574,
      "learning_rate": 1e-06,
      "logits/chosen": 24.683202743530273,
      "logits/rejected": 23.952590942382812,
      "logps/chosen": -409.4302673339844,
      "logps/rejected": -463.2736511230469,
      "loss": 0.2483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2341659516096115,
      "rewards/margins": 0.48990583419799805,
      "rewards/rejected": -0.25573989748954773,
      "step": 78
    },
    {
      "debug/policy_chosen_logits": 26.540374755859375,
      "debug/policy_chosen_logps": -422.86810302734375,
      "debug/policy_rejected_logits": 30.67388916015625,
      "debug/policy_rejected_logps": -492.8260192871094,
      "debug/reference_chosen_logps": -438.4207763671875,
      "debug/reference_rejected_logps": -458.2353515625,
      "epoch": 0.8681318681318682,
      "grad_norm": 5.889314624750013,
      "learning_rate": 1e-06,
      "logits/chosen": 26.540374755859375,
      "logits/rejected": 30.67388916015625,
      "logps/chosen": -422.86810302734375,
      "logps/rejected": -492.8260192871094,
      "loss": 0.264,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15552687644958496,
      "rewards/margins": 0.5014333724975586,
      "rewards/rejected": -0.34590649604797363,
      "step": 79
    },
    {
      "debug/policy_chosen_logits": 24.57747459411621,
      "debug/policy_chosen_logps": -390.2710266113281,
      "debug/policy_rejected_logits": 27.390230178833008,
      "debug/policy_rejected_logps": -489.9441223144531,
      "debug/reference_chosen_logps": -416.788818359375,
      "debug/reference_rejected_logps": -440.2818298339844,
      "epoch": 0.8791208791208791,
      "grad_norm": 4.745834840744495,
      "learning_rate": 1e-06,
      "logits/chosen": 24.57747459411621,
      "logits/rejected": 27.390230178833008,
      "logps/chosen": -390.2710266113281,
      "logps/rejected": -489.9441223144531,
      "loss": 0.2894,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2651780843734741,
      "rewards/margins": 0.76180100440979,
      "rewards/rejected": -0.49662283062934875,
      "step": 80
    },
    {
      "debug/policy_chosen_logits": 25.34842300415039,
      "debug/policy_chosen_logps": -390.0006103515625,
      "debug/policy_rejected_logits": 27.055194854736328,
      "debug/policy_rejected_logps": -460.668701171875,
      "debug/reference_chosen_logps": -408.5838623046875,
      "debug/reference_rejected_logps": -444.5193786621094,
      "epoch": 0.8901098901098901,
      "grad_norm": 4.590683530349925,
      "learning_rate": 1e-06,
      "logits/chosen": 25.34842300415039,
      "logits/rejected": 27.055194854736328,
      "logps/chosen": -390.0006103515625,
      "logps/rejected": -460.668701171875,
      "loss": 0.2902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1858326643705368,
      "rewards/margins": 0.3473258316516876,
      "rewards/rejected": -0.16149315237998962,
      "step": 81
    },
    {
      "debug/policy_chosen_logits": 27.74835968017578,
      "debug/policy_chosen_logps": -413.76629638671875,
      "debug/policy_rejected_logits": 27.351459503173828,
      "debug/policy_rejected_logps": -438.90216064453125,
      "debug/reference_chosen_logps": -443.37615966796875,
      "debug/reference_rejected_logps": -417.8377380371094,
      "epoch": 0.9010989010989011,
      "grad_norm": 4.49887483206356,
      "learning_rate": 1e-06,
      "logits/chosen": 27.74835968017578,
      "logits/rejected": 27.351459503173828,
      "logps/chosen": -413.76629638671875,
      "logps/rejected": -438.90216064453125,
      "loss": 0.252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29609841108322144,
      "rewards/margins": 0.5067427754402161,
      "rewards/rejected": -0.21064436435699463,
      "step": 82
    },
    {
      "debug/policy_chosen_logits": 26.60724639892578,
      "debug/policy_chosen_logps": -380.8109130859375,
      "debug/policy_rejected_logits": 30.683988571166992,
      "debug/policy_rejected_logps": -487.4014892578125,
      "debug/reference_chosen_logps": -419.3782958984375,
      "debug/reference_rejected_logps": -462.61505126953125,
      "epoch": 0.9120879120879121,
      "grad_norm": 4.452776489719901,
      "learning_rate": 1e-06,
      "logits/chosen": 26.60724639892578,
      "logits/rejected": 30.683988571166992,
      "logps/chosen": -380.8109130859375,
      "logps/rejected": -487.4014892578125,
      "loss": 0.281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38567405939102173,
      "rewards/margins": 0.6335387825965881,
      "rewards/rejected": -0.2478647232055664,
      "step": 83
    },
    {
      "debug/policy_chosen_logits": 23.03528594970703,
      "debug/policy_chosen_logps": -416.22314453125,
      "debug/policy_rejected_logits": 23.581472396850586,
      "debug/policy_rejected_logps": -434.1588439941406,
      "debug/reference_chosen_logps": -440.158935546875,
      "debug/reference_rejected_logps": -421.5128173828125,
      "epoch": 0.9230769230769231,
      "grad_norm": 4.971404649364657,
      "learning_rate": 1e-06,
      "logits/chosen": 23.03528594970703,
      "logits/rejected": 23.581472396850586,
      "logps/chosen": -416.22314453125,
      "logps/rejected": -434.1588439941406,
      "loss": 0.2359,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23935794830322266,
      "rewards/margins": 0.36581796407699585,
      "rewards/rejected": -0.1264600157737732,
      "step": 84
    },
    {
      "debug/policy_chosen_logits": 26.0145320892334,
      "debug/policy_chosen_logps": -407.3331298828125,
      "debug/policy_rejected_logits": 22.86039924621582,
      "debug/policy_rejected_logps": -455.66888427734375,
      "debug/reference_chosen_logps": -429.9805908203125,
      "debug/reference_rejected_logps": -431.9481201171875,
      "epoch": 0.9340659340659341,
      "grad_norm": 4.560309498305691,
      "learning_rate": 1e-06,
      "logits/chosen": 26.0145320892334,
      "logits/rejected": 22.86039924621582,
      "logps/chosen": -407.3331298828125,
      "logps/rejected": -455.66888427734375,
      "loss": 0.2926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22647437453269958,
      "rewards/margins": 0.46368181705474854,
      "rewards/rejected": -0.23720744252204895,
      "step": 85
    },
    {
      "debug/policy_chosen_logits": 21.526865005493164,
      "debug/policy_chosen_logps": -391.841552734375,
      "debug/policy_rejected_logits": 25.782846450805664,
      "debug/policy_rejected_logps": -485.3880310058594,
      "debug/reference_chosen_logps": -410.5163269042969,
      "debug/reference_rejected_logps": -450.18896484375,
      "epoch": 0.945054945054945,
      "grad_norm": 3.115000092835439,
      "learning_rate": 1e-06,
      "logits/chosen": 21.526865005493164,
      "logits/rejected": 25.782846450805664,
      "logps/chosen": -391.841552734375,
      "logps/rejected": -485.3880310058594,
      "loss": 0.227,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1867476999759674,
      "rewards/margins": 0.5387383699417114,
      "rewards/rejected": -0.35199064016342163,
      "step": 86
    },
    {
      "debug/policy_chosen_logits": 24.497520446777344,
      "debug/policy_chosen_logps": -430.42303466796875,
      "debug/policy_rejected_logits": 26.349315643310547,
      "debug/policy_rejected_logps": -465.03375244140625,
      "debug/reference_chosen_logps": -443.20037841796875,
      "debug/reference_rejected_logps": -455.8772888183594,
      "epoch": 0.9560439560439561,
      "grad_norm": 4.513341277064094,
      "learning_rate": 1e-06,
      "logits/chosen": 24.497520446777344,
      "logits/rejected": 26.349315643310547,
      "logps/chosen": -430.42303466796875,
      "logps/rejected": -465.03375244140625,
      "loss": 0.2854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12777358293533325,
      "rewards/margins": 0.2193382978439331,
      "rewards/rejected": -0.09156470745801926,
      "step": 87
    },
    {
      "debug/policy_chosen_logits": 18.35858917236328,
      "debug/policy_chosen_logps": -389.84100341796875,
      "debug/policy_rejected_logits": 21.731107711791992,
      "debug/policy_rejected_logps": -423.7160339355469,
      "debug/reference_chosen_logps": -406.00408935546875,
      "debug/reference_rejected_logps": -423.71820068359375,
      "epoch": 0.967032967032967,
      "grad_norm": 4.8398310648982354,
      "learning_rate": 1e-06,
      "logits/chosen": 18.35858917236328,
      "logits/rejected": 21.731107711791992,
      "logps/chosen": -389.84100341796875,
      "logps/rejected": -423.7160339355469,
      "loss": 0.2595,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16163063049316406,
      "rewards/margins": 0.16160908341407776,
      "rewards/rejected": 2.155173569917679e-05,
      "step": 88
    },
    {
      "debug/policy_chosen_logits": 23.2659969329834,
      "debug/policy_chosen_logps": -423.4161376953125,
      "debug/policy_rejected_logits": 28.146160125732422,
      "debug/policy_rejected_logps": -473.67059326171875,
      "debug/reference_chosen_logps": -441.9681091308594,
      "debug/reference_rejected_logps": -431.1749267578125,
      "epoch": 0.978021978021978,
      "grad_norm": 6.467539115961516,
      "learning_rate": 1e-06,
      "logits/chosen": 23.2659969329834,
      "logits/rejected": 28.146160125732422,
      "logps/chosen": -423.4161376953125,
      "logps/rejected": -473.67059326171875,
      "loss": 0.2244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1855195164680481,
      "rewards/margins": 0.610476553440094,
      "rewards/rejected": -0.4249570369720459,
      "step": 89
    },
    {
      "debug/policy_chosen_logits": 27.019420623779297,
      "debug/policy_chosen_logps": -413.4447021484375,
      "debug/policy_rejected_logits": 31.003211975097656,
      "debug/policy_rejected_logps": -500.8101501464844,
      "debug/reference_chosen_logps": -434.320068359375,
      "debug/reference_rejected_logps": -467.6002502441406,
      "epoch": 0.989010989010989,
      "grad_norm": 4.1029333690791745,
      "learning_rate": 1e-06,
      "logits/chosen": 27.019420623779297,
      "logits/rejected": 31.003211975097656,
      "logps/chosen": -413.4447021484375,
      "logps/rejected": -500.8101501464844,
      "loss": 0.2992,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20875364542007446,
      "rewards/margins": 0.5408526062965393,
      "rewards/rejected": -0.33209896087646484,
      "step": 90
    },
    {
      "debug/policy_chosen_logits": 29.092975616455078,
      "debug/policy_chosen_logps": -404.4205017089844,
      "debug/policy_rejected_logits": 23.611309051513672,
      "debug/policy_rejected_logps": -429.15985107421875,
      "debug/reference_chosen_logps": -424.5293884277344,
      "debug/reference_rejected_logps": -418.74169921875,
      "epoch": 1.0,
      "grad_norm": 5.419471225413546,
      "learning_rate": 1e-06,
      "logits/chosen": 29.092975616455078,
      "logits/rejected": 23.611309051513672,
      "logps/chosen": -404.4205017089844,
      "logps/rejected": -429.15985107421875,
      "loss": 0.2937,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20108896493911743,
      "rewards/margins": 0.30527064204216003,
      "rewards/rejected": -0.10418166220188141,
      "step": 91
    },
    {
      "epoch": 1.0,
      "step": 91,
      "total_flos": 0.0,
      "train_loss": 0.3198396195094664,
      "train_runtime": 711.7161,
      "train_samples_per_second": 8.179,
      "train_steps_per_second": 0.128
    }
  ],
  "logging_steps": 1,
  "max_steps": 91,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}