trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9874476987447699,
  "eval_steps": 500,
  "global_step": 59,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.016736401673640166,
      "grad_norm": 11.812123208113574,
      "learning_rate": 8.333333333333333e-08,
      "logits/chosen": -2.834899425506592,
      "logits/rejected": -2.7613072395324707,
      "logps/chosen": -203.2696075439453,
      "logps/pi_response": -114.11784362792969,
      "logps/ref_response": -114.11784362792969,
      "logps/rejected": -216.69369506835938,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.16736401673640167,
      "grad_norm": 12.187784364287845,
      "learning_rate": 4.930057285201027e-07,
      "logits/chosen": -2.752955436706543,
      "logits/rejected": -2.7144250869750977,
      "logps/chosen": -213.47360229492188,
      "logps/pi_response": -115.57353973388672,
      "logps/ref_response": -114.95613861083984,
      "logps/rejected": -260.9118347167969,
      "loss": 0.6851,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.035287365317344666,
      "rewards/margins": 0.021063316613435745,
      "rewards/rejected": -0.05635068938136101,
      "step": 10
    },
    {
      "epoch": 0.33472803347280333,
      "grad_norm": 26.05433637861999,
      "learning_rate": 4.187457503795526e-07,
      "logits/chosen": -2.769129514694214,
      "logits/rejected": -2.730208158493042,
      "logps/chosen": -251.2847137451172,
      "logps/pi_response": -112.8050308227539,
      "logps/ref_response": -115.5127182006836,
      "logps/rejected": -303.66827392578125,
      "loss": 0.6307,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": -0.2332080900669098,
      "rewards/margins": 0.2652435004711151,
      "rewards/rejected": -0.4984516203403473,
      "step": 20
    },
    {
      "epoch": 0.502092050209205,
      "grad_norm": 16.251673604099125,
      "learning_rate": 2.8691164100062034e-07,
      "logits/chosen": -2.7468676567077637,
      "logits/rejected": -2.7079989910125732,
      "logps/chosen": -293.1772766113281,
      "logps/pi_response": -142.03948974609375,
      "logps/ref_response": -120.54533386230469,
      "logps/rejected": -340.1116638183594,
      "loss": 0.5935,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -0.5449068546295166,
      "rewards/margins": 0.3869662284851074,
      "rewards/rejected": -0.9318730235099792,
      "step": 30
    },
    {
      "epoch": 0.6694560669456067,
      "grad_norm": 15.358545526706314,
      "learning_rate": 1.4248369943086995e-07,
      "logits/chosen": -2.662774085998535,
      "logits/rejected": -2.6290879249572754,
      "logps/chosen": -267.100341796875,
      "logps/pi_response": -145.8026580810547,
      "logps/ref_response": -112.255859375,
      "logps/rejected": -363.7671203613281,
      "loss": 0.555,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.5281414985656738,
      "rewards/margins": 0.5264478921890259,
      "rewards/rejected": -1.0545893907546997,
      "step": 40
    },
    {
      "epoch": 0.8368200836820083,
      "grad_norm": 16.785486572836216,
      "learning_rate": 3.473909705816111e-08,
      "logits/chosen": -2.6454825401306152,
      "logits/rejected": -2.613494396209717,
      "logps/chosen": -275.4294128417969,
      "logps/pi_response": -153.84144592285156,
      "logps/ref_response": -113.73796081542969,
      "logps/rejected": -336.74237060546875,
      "loss": 0.5511,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.5632938742637634,
      "rewards/margins": 0.45978373289108276,
      "rewards/rejected": -1.0230777263641357,
      "step": 50
    },
    {
      "epoch": 0.9874476987447699,
      "step": 59,
      "total_flos": 0.0,
      "train_loss": 0.5946699562719313,
      "train_runtime": 2712.221,
      "train_samples_per_second": 5.635,
      "train_steps_per_second": 0.022
    }
  ],
  "logging_steps": 10,
  "max_steps": 59,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}