| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.9748953974895398, |
| "eval_steps": 500, |
| "global_step": 118, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.016736401673640166, |
| "grad_norm": 103.69568207506418, |
| "learning_rate": 4.166666666666666e-08, |
| "logits/chosen": -1.9738563299179077, |
| "logits/rejected": -1.8741222620010376, |
| "logps/chosen": -277.1735534667969, |
| "logps/pi_response": -172.42257690429688, |
| "logps/ref_response": -172.42257690429688, |
| "logps/rejected": -634.8126220703125, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.16736401673640167, |
| "grad_norm": 50.929253150896805, |
| "learning_rate": 4.1666666666666667e-07, |
| "logits/chosen": -1.9078360795974731, |
| "logits/rejected": -1.8270505666732788, |
| "logps/chosen": -296.2008056640625, |
| "logps/pi_response": -164.65350341796875, |
| "logps/ref_response": -161.74786376953125, |
| "logps/rejected": -624.4210815429688, |
| "loss": 0.6527, |
| "rewards/accuracies": 0.6354166865348816, |
| "rewards/chosen": -0.08622797578573227, |
| "rewards/margins": 0.11728954315185547, |
| "rewards/rejected": -0.20351749658584595, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.33472803347280333, |
| "grad_norm": 82.32138711629948, |
| "learning_rate": 4.930057285201027e-07, |
| "logits/chosen": -1.6078555583953857, |
| "logits/rejected": -1.4752347469329834, |
| "logps/chosen": -472.5746154785156, |
| "logps/pi_response": -237.7078399658203, |
| "logps/ref_response": -164.33425903320312, |
| "logps/rejected": -879.9568481445312, |
| "loss": 0.6118, |
| "rewards/accuracies": 0.7406250238418579, |
| "rewards/chosen": -1.5813623666763306, |
| "rewards/margins": 1.3661245107650757, |
| "rewards/rejected": -2.947486639022827, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.502092050209205, |
| "grad_norm": 64.12887587578865, |
| "learning_rate": 4.652609029418388e-07, |
| "logits/chosen": -1.5715327262878418, |
| "logits/rejected": -1.41808021068573, |
| "logps/chosen": -427.7864685058594, |
| "logps/pi_response": -249.6379852294922, |
| "logps/ref_response": -163.8944854736328, |
| "logps/rejected": -813.6851196289062, |
| "loss": 0.5092, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.9770371317863464, |
| "rewards/margins": 1.3367812633514404, |
| "rewards/rejected": -2.3138184547424316, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.6694560669456067, |
| "grad_norm": 57.465221111933175, |
| "learning_rate": 4.187457503795526e-07, |
| "logits/chosen": -1.0906447172164917, |
| "logits/rejected": -0.6990022659301758, |
| "logps/chosen": -382.23626708984375, |
| "logps/pi_response": -218.41738891601562, |
| "logps/ref_response": -167.90982055664062, |
| "logps/rejected": -835.4563598632812, |
| "loss": 0.4761, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -0.8820472955703735, |
| "rewards/margins": 1.2603753805160522, |
| "rewards/rejected": -2.142422676086426, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.8368200836820083, |
| "grad_norm": 64.61034301359892, |
| "learning_rate": 3.5751630056913013e-07, |
| "logits/chosen": -0.3525225520133972, |
| "logits/rejected": 0.21344999969005585, |
| "logps/chosen": -453.05670166015625, |
| "logps/pi_response": -244.5023956298828, |
| "logps/ref_response": -162.8319549560547, |
| "logps/rejected": -875.3497924804688, |
| "loss": 0.4293, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -1.3764469623565674, |
| "rewards/margins": 1.3408019542694092, |
| "rewards/rejected": -2.7172489166259766, |
| "step": 50 |
| }, |
| { |
| "epoch": 1.00418410041841, |
| "grad_norm": 52.83213167715229, |
| "learning_rate": 2.8691164100062034e-07, |
| "logits/chosen": -0.08212046325206757, |
| "logits/rejected": 0.40920084714889526, |
| "logps/chosen": -454.02728271484375, |
| "logps/pi_response": -256.43280029296875, |
| "logps/ref_response": -165.61346435546875, |
| "logps/rejected": -875.42431640625, |
| "loss": 0.4067, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -1.3304615020751953, |
| "rewards/margins": 1.3796277046203613, |
| "rewards/rejected": -2.7100892066955566, |
| "step": 60 |
| }, |
| { |
| "epoch": 1.1715481171548117, |
| "grad_norm": 32.183217671820124, |
| "learning_rate": 2.1308835899937972e-07, |
| "logits/chosen": 0.11680100858211517, |
| "logits/rejected": 0.8223470449447632, |
| "logps/chosen": -457.57977294921875, |
| "logps/pi_response": -270.11724853515625, |
| "logps/ref_response": -166.54843139648438, |
| "logps/rejected": -930.5846557617188, |
| "loss": 0.3015, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -1.324484944343567, |
| "rewards/margins": 1.940722107887268, |
| "rewards/rejected": -3.265206813812256, |
| "step": 70 |
| }, |
| { |
| "epoch": 1.3389121338912133, |
| "grad_norm": 25.86830503289715, |
| "learning_rate": 1.4248369943086995e-07, |
| "logits/chosen": 0.35845622420310974, |
| "logits/rejected": 1.1116231679916382, |
| "logps/chosen": -412.78466796875, |
| "logps/pi_response": -262.92047119140625, |
| "logps/ref_response": -153.2851104736328, |
| "logps/rejected": -979.3216552734375, |
| "loss": 0.2724, |
| "rewards/accuracies": 0.909375011920929, |
| "rewards/chosen": -1.3652256727218628, |
| "rewards/margins": 2.411055326461792, |
| "rewards/rejected": -3.7762808799743652, |
| "step": 80 |
| }, |
| { |
| "epoch": 1.506276150627615, |
| "grad_norm": 27.727359766057667, |
| "learning_rate": 8.125424962044741e-08, |
| "logits/chosen": 0.12312891334295273, |
| "logits/rejected": 1.0509188175201416, |
| "logps/chosen": -418.90350341796875, |
| "logps/pi_response": -270.42474365234375, |
| "logps/ref_response": -164.84889221191406, |
| "logps/rejected": -976.3717651367188, |
| "loss": 0.2817, |
| "rewards/accuracies": 0.8968750238418579, |
| "rewards/chosen": -1.3233258724212646, |
| "rewards/margins": 2.3037142753601074, |
| "rewards/rejected": -3.627040386199951, |
| "step": 90 |
| }, |
| { |
| "epoch": 1.6736401673640167, |
| "grad_norm": 40.83751123410407, |
| "learning_rate": 3.473909705816111e-08, |
| "logits/chosen": 0.3033740520477295, |
| "logits/rejected": 1.2026764154434204, |
| "logps/chosen": -428.5899353027344, |
| "logps/pi_response": -274.1754150390625, |
| "logps/ref_response": -159.84397888183594, |
| "logps/rejected": -980.5960083007812, |
| "loss": 0.2724, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -1.342337727546692, |
| "rewards/margins": 2.3560409545898438, |
| "rewards/rejected": -3.698378801345825, |
| "step": 100 |
| }, |
| { |
| "epoch": 1.8410041841004183, |
| "grad_norm": 68.33807903105533, |
| "learning_rate": 6.994271479897313e-09, |
| "logits/chosen": 0.32156121730804443, |
| "logits/rejected": 1.186035394668579, |
| "logps/chosen": -485.846435546875, |
| "logps/pi_response": -276.8388671875, |
| "logps/ref_response": -163.15513610839844, |
| "logps/rejected": -959.0426635742188, |
| "loss": 0.2826, |
| "rewards/accuracies": 0.8843749761581421, |
| "rewards/chosen": -1.5259841680526733, |
| "rewards/margins": 2.120156764984131, |
| "rewards/rejected": -3.6461405754089355, |
| "step": 110 |
| }, |
| { |
| "epoch": 1.9748953974895398, |
| "step": 118, |
| "total_flos": 0.0, |
| "train_loss": 0.4021359480033487, |
| "train_runtime": 5275.9273, |
| "train_samples_per_second": 5.793, |
| "train_steps_per_second": 0.022 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 118, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 2, |
| "save_steps": 100, |
| "total_flos": 0.0, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
