{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.9748953974895398,
  "eval_steps": 500,
  "global_step": 118,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.016736401673640166,
      "grad_norm": 103.69568207506418,
      "learning_rate": 4.166666666666666e-08,
      "logits/chosen": -1.9738563299179077,
      "logits/rejected": -1.8741222620010376,
      "logps/chosen": -277.1735534667969,
      "logps/pi_response": -172.42257690429688,
      "logps/ref_response": -172.42257690429688,
      "logps/rejected": -634.8126220703125,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.16736401673640167,
      "grad_norm": 50.929253150896805,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -1.9078360795974731,
      "logits/rejected": -1.8270505666732788,
      "logps/chosen": -296.2008056640625,
      "logps/pi_response": -164.65350341796875,
      "logps/ref_response": -161.74786376953125,
      "logps/rejected": -624.4210815429688,
      "loss": 0.6527,
      "rewards/accuracies": 0.6354166865348816,
      "rewards/chosen": -0.08622797578573227,
      "rewards/margins": 0.11728954315185547,
      "rewards/rejected": -0.20351749658584595,
      "step": 10
    },
    {
      "epoch": 0.33472803347280333,
      "grad_norm": 82.32138711629948,
      "learning_rate": 4.930057285201027e-07,
      "logits/chosen": -1.6078555583953857,
      "logits/rejected": -1.4752347469329834,
      "logps/chosen": -472.5746154785156,
      "logps/pi_response": -237.7078399658203,
      "logps/ref_response": -164.33425903320312,
      "logps/rejected": -879.9568481445312,
      "loss": 0.6118,
      "rewards/accuracies": 0.7406250238418579,
      "rewards/chosen": -1.5813623666763306,
      "rewards/margins": 1.3661245107650757,
      "rewards/rejected": -2.947486639022827,
      "step": 20
    },
    {
      "epoch": 0.502092050209205,
      "grad_norm": 64.12887587578865,
      "learning_rate": 4.652609029418388e-07,
      "logits/chosen": -1.5715327262878418,
      "logits/rejected": -1.41808021068573,
      "logps/chosen": -427.7864685058594,
      "logps/pi_response": -249.6379852294922,
      "logps/ref_response": -163.8944854736328,
      "logps/rejected": -813.6851196289062,
      "loss": 0.5092,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9770371317863464,
      "rewards/margins": 1.3367812633514404,
      "rewards/rejected": -2.3138184547424316,
      "step": 30
    },
    {
      "epoch": 0.6694560669456067,
      "grad_norm": 57.465221111933175,
      "learning_rate": 4.187457503795526e-07,
      "logits/chosen": -1.0906447172164917,
      "logits/rejected": -0.6990022659301758,
      "logps/chosen": -382.23626708984375,
      "logps/pi_response": -218.41738891601562,
      "logps/ref_response": -167.90982055664062,
      "logps/rejected": -835.4563598632812,
      "loss": 0.4761,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.8820472955703735,
      "rewards/margins": 1.2603753805160522,
      "rewards/rejected": -2.142422676086426,
      "step": 40
    },
    {
      "epoch": 0.8368200836820083,
      "grad_norm": 64.61034301359892,
      "learning_rate": 3.5751630056913013e-07,
      "logits/chosen": -0.3525225520133972,
      "logits/rejected": 0.21344999969005585,
      "logps/chosen": -453.05670166015625,
      "logps/pi_response": -244.5023956298828,
      "logps/ref_response": -162.8319549560547,
      "logps/rejected": -875.3497924804688,
      "loss": 0.4293,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.3764469623565674,
      "rewards/margins": 1.3408019542694092,
      "rewards/rejected": -2.7172489166259766,
      "step": 50
    },
    {
      "epoch": 1.00418410041841,
      "grad_norm": 52.83213167715229,
      "learning_rate": 2.8691164100062034e-07,
      "logits/chosen": -0.08212046325206757,
      "logits/rejected": 0.40920084714889526,
      "logps/chosen": -454.02728271484375,
      "logps/pi_response": -256.43280029296875,
      "logps/ref_response": -165.61346435546875,
      "logps/rejected": -875.42431640625,
      "loss": 0.4067,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.3304615020751953,
      "rewards/margins": 1.3796277046203613,
      "rewards/rejected": -2.7100892066955566,
      "step": 60
    },
    {
      "epoch": 1.1715481171548117,
      "grad_norm": 32.183217671820124,
      "learning_rate": 2.1308835899937972e-07,
      "logits/chosen": 0.11680100858211517,
      "logits/rejected": 0.8223470449447632,
      "logps/chosen": -457.57977294921875,
      "logps/pi_response": -270.11724853515625,
      "logps/ref_response": -166.54843139648438,
      "logps/rejected": -930.5846557617188,
      "loss": 0.3015,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.324484944343567,
      "rewards/margins": 1.940722107887268,
      "rewards/rejected": -3.265206813812256,
      "step": 70
    },
    {
      "epoch": 1.3389121338912133,
      "grad_norm": 25.86830503289715,
      "learning_rate": 1.4248369943086995e-07,
      "logits/chosen": 0.35845622420310974,
      "logits/rejected": 1.1116231679916382,
      "logps/chosen": -412.78466796875,
      "logps/pi_response": -262.92047119140625,
      "logps/ref_response": -153.2851104736328,
      "logps/rejected": -979.3216552734375,
      "loss": 0.2724,
      "rewards/accuracies": 0.909375011920929,
      "rewards/chosen": -1.3652256727218628,
      "rewards/margins": 2.411055326461792,
      "rewards/rejected": -3.7762808799743652,
      "step": 80
    },
    {
      "epoch": 1.506276150627615,
      "grad_norm": 27.727359766057667,
      "learning_rate": 8.125424962044741e-08,
      "logits/chosen": 0.12312891334295273,
      "logits/rejected": 1.0509188175201416,
      "logps/chosen": -418.90350341796875,
      "logps/pi_response": -270.42474365234375,
      "logps/ref_response": -164.84889221191406,
      "logps/rejected": -976.3717651367188,
      "loss": 0.2817,
      "rewards/accuracies": 0.8968750238418579,
      "rewards/chosen": -1.3233258724212646,
      "rewards/margins": 2.3037142753601074,
      "rewards/rejected": -3.627040386199951,
      "step": 90
    },
    {
      "epoch": 1.6736401673640167,
      "grad_norm": 40.83751123410407,
      "learning_rate": 3.473909705816111e-08,
      "logits/chosen": 0.3033740520477295,
      "logits/rejected": 1.2026764154434204,
      "logps/chosen": -428.5899353027344,
      "logps/pi_response": -274.1754150390625,
      "logps/ref_response": -159.84397888183594,
      "logps/rejected": -980.5960083007812,
      "loss": 0.2724,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.342337727546692,
      "rewards/margins": 2.3560409545898438,
      "rewards/rejected": -3.698378801345825,
      "step": 100
    },
    {
      "epoch": 1.8410041841004183,
      "grad_norm": 68.33807903105533,
      "learning_rate": 6.994271479897313e-09,
      "logits/chosen": 0.32156121730804443,
      "logits/rejected": 1.186035394668579,
      "logps/chosen": -485.846435546875,
      "logps/pi_response": -276.8388671875,
      "logps/ref_response": -163.15513610839844,
      "logps/rejected": -959.0426635742188,
      "loss": 0.2826,
      "rewards/accuracies": 0.8843749761581421,
      "rewards/chosen": -1.5259841680526733,
      "rewards/margins": 2.120156764984131,
      "rewards/rejected": -3.6461405754089355,
      "step": 110
    },
    {
      "epoch": 1.9748953974895398,
      "step": 118,
      "total_flos": 0.0,
      "train_loss": 0.4021359480033487,
      "train_runtime": 5275.9273,
      "train_samples_per_second": 5.793,
      "train_steps_per_second": 0.022
    }
  ],
  "logging_steps": 10,
  "max_steps": 118,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}