| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.0, |
| "eval_steps": 195, |
| "global_step": 976, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "approx. KL/chosen": 0.0, |
| "approx. KL/rejected": 0.0, |
| "epoch": 0.0010245901639344263, |
| "grad_norm": 98.9356918334961, |
| "learning_rate": 0.0, |
| "logp/chosen": -956.0, |
| "logp/rejected": -1216.0, |
| "loss": 1.31298828125, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.0, |
| "reward/chosen": 0.0, |
| "reward/margin": 0.0, |
| "reward/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "approx. KL/chosen": 0.0, |
| "approx. KL/rejected": 0.0, |
| "epoch": 0.0020491803278688526, |
| "grad_norm": 99.47376251220703, |
| "learning_rate": 8.163265306122448e-08, |
| "logp/chosen": -984.0, |
| "logp/rejected": -1000.0, |
| "loss": 1.21484375, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.0, |
| "reward/chosen": 0.0, |
| "reward/margin": 0.0, |
| "reward/rejected": 0.0, |
| "step": 2 |
| }, |
| { |
| "approx. KL/chosen": 4.0625, |
| "approx. KL/rejected": 8.0625, |
| "epoch": 0.0030737704918032786, |
| "grad_norm": 103.22154235839844, |
| "learning_rate": 1.6326530612244896e-07, |
| "logp/chosen": -1112.0, |
| "logp/rejected": -1232.0, |
| "loss": 1.3173828125, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -0.2255859375, |
| "reward/margin": 0.2001953125, |
| "reward/rejected": -0.42578125, |
| "step": 3 |
| }, |
| { |
| "approx. KL/chosen": 1.5, |
| "approx. KL/rejected": 0.06640625, |
| "epoch": 0.004098360655737705, |
| "grad_norm": 86.62486267089844, |
| "learning_rate": 2.4489795918367347e-07, |
| "logp/chosen": -644.0, |
| "logp/rejected": -840.0, |
| "loss": 1.2587890625, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": 0.0, |
| "reward/margin": 0.03125, |
| "reward/rejected": -0.03125, |
| "step": 4 |
| }, |
| { |
| "approx. KL/chosen": 4.25, |
| "approx. KL/rejected": 5.0, |
| "epoch": 0.005122950819672131, |
| "grad_norm": 148.9287109375, |
| "learning_rate": 3.265306122448979e-07, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1728.0, |
| "loss": 1.5703125, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.125, |
| "reward/chosen": -0.25, |
| "reward/margin": -0.349609375, |
| "reward/rejected": 0.10009765625, |
| "step": 5 |
| }, |
| { |
| "approx. KL/chosen": 1.25, |
| "approx. KL/rejected": 21.25, |
| "epoch": 0.006147540983606557, |
| "grad_norm": 144.8015594482422, |
| "learning_rate": 4.0816326530612243e-07, |
| "logp/chosen": -896.0, |
| "logp/rejected": -1224.0, |
| "loss": 1.720703125, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.125, |
| "reward/chosen": 0.150390625, |
| "reward/margin": -0.6015625, |
| "reward/rejected": 0.75, |
| "step": 6 |
| }, |
| { |
| "approx. KL/chosen": 4.0, |
| "approx. KL/rejected": 4.25, |
| "epoch": 0.007172131147540984, |
| "grad_norm": 91.25936889648438, |
| "learning_rate": 4.897959183673469e-07, |
| "logp/chosen": -804.0, |
| "logp/rejected": -824.0, |
| "loss": 1.08349609375, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": 0.2001953125, |
| "reward/margin": 0.462890625, |
| "reward/rejected": -0.263671875, |
| "step": 7 |
| }, |
| { |
| "approx. KL/chosen": 1.25, |
| "approx. KL/rejected": 0.25, |
| "epoch": 0.00819672131147541, |
| "grad_norm": 87.2760238647461, |
| "learning_rate": 5.714285714285714e-07, |
| "logp/chosen": -716.0, |
| "logp/rejected": -880.0, |
| "loss": 1.18701171875, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": 0.15625, |
| "reward/margin": 0.1064453125, |
| "reward/rejected": 0.050048828125, |
| "step": 8 |
| }, |
| { |
| "approx. KL/chosen": 1.515625, |
| "approx. KL/rejected": 1.0, |
| "epoch": 0.009221311475409836, |
| "grad_norm": 80.08042907714844, |
| "learning_rate": 6.530612244897958e-07, |
| "logp/chosen": -584.0, |
| "logp/rejected": -716.0, |
| "loss": 1.2099609375, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": 0.1875, |
| "reward/margin": 0.08740234375, |
| "reward/rejected": 0.10009765625, |
| "step": 9 |
| }, |
| { |
| "approx. KL/chosen": 5.0, |
| "approx. KL/rejected": 8.0, |
| "epoch": 0.010245901639344262, |
| "grad_norm": 131.06582641601562, |
| "learning_rate": 7.346938775510204e-07, |
| "logp/chosen": -1128.0, |
| "logp/rejected": -1656.0, |
| "loss": 1.29931640625, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 0.30078125, |
| "reward/rejected": -0.400390625, |
| "step": 10 |
| }, |
| { |
| "approx. KL/chosen": 1.0, |
| "approx. KL/rejected": 0.0625, |
| "epoch": 0.011270491803278689, |
| "grad_norm": 106.2105712890625, |
| "learning_rate": 8.163265306122449e-07, |
| "logp/chosen": -904.0, |
| "logp/rejected": -1012.0, |
| "loss": 1.33349609375, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.125, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": -0.0751953125, |
| "reward/rejected": -0.0250244140625, |
| "step": 11 |
| }, |
| { |
| "approx. KL/chosen": 20.0, |
| "approx. KL/rejected": 0.25, |
| "epoch": 0.012295081967213115, |
| "grad_norm": 74.43769073486328, |
| "learning_rate": 8.979591836734693e-07, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1320.0, |
| "loss": 1.1142578125, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": 0.6015625, |
| "reward/margin": 0.65234375, |
| "reward/rejected": -0.050048828125, |
| "step": 12 |
| }, |
| { |
| "approx. KL/chosen": 0.5, |
| "approx. KL/rejected": 1.0, |
| "epoch": 0.01331967213114754, |
| "grad_norm": 101.6725845336914, |
| "learning_rate": 9.795918367346939e-07, |
| "logp/chosen": -932.0, |
| "logp/rejected": -1016.0, |
| "loss": 1.3740234375, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.125, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": -0.1943359375, |
| "reward/rejected": 0.09375, |
| "step": 13 |
| }, |
| { |
| "approx. KL/chosen": 2.0, |
| "approx. KL/rejected": 0.0, |
| "epoch": 0.014344262295081968, |
| "grad_norm": 111.85306549072266, |
| "learning_rate": 1.0612244897959184e-06, |
| "logp/chosen": -1184.0, |
| "logp/rejected": -1328.0, |
| "loss": 1.283203125, |
| "nll_loss": 0.671875, |
| "reward/accuracy": 0.25, |
| "reward/chosen": 0.2001953125, |
| "reward/margin": 0.2001953125, |
| "reward/rejected": 0.0, |
| "step": 14 |
| }, |
| { |
| "approx. KL/chosen": 2.0, |
| "approx. KL/rejected": 1.25, |
| "epoch": 0.015368852459016393, |
| "grad_norm": 93.85140991210938, |
| "learning_rate": 1.1428571428571428e-06, |
| "logp/chosen": -816.0, |
| "logp/rejected": -664.0, |
| "loss": 1.4052734375, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.0, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": -0.25, |
| "reward/rejected": 0.050048828125, |
| "step": 15 |
| }, |
| { |
| "approx. KL/chosen": 8.0, |
| "approx. KL/rejected": 13.0, |
| "epoch": 0.01639344262295082, |
| "grad_norm": 91.65290832519531, |
| "learning_rate": 1.2244897959183673e-06, |
| "logp/chosen": -928.0, |
| "logp/rejected": -1136.0, |
| "loss": 1.25244140625, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": 0.0, |
| "reward/margin": 0.5, |
| "reward/rejected": -0.5, |
| "step": 16 |
| }, |
| { |
| "approx. KL/chosen": 8.3125, |
| "approx. KL/rejected": 17.0, |
| "epoch": 0.017418032786885244, |
| "grad_norm": 95.5101318359375, |
| "learning_rate": 1.3061224489795917e-06, |
| "logp/chosen": -976.0, |
| "logp/rejected": -940.0, |
| "loss": 1.455078125, |
| "nll_loss": 0.6640625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.375, |
| "reward/margin": 0.140625, |
| "reward/rejected": -0.515625, |
| "step": 17 |
| }, |
| { |
| "approx. KL/chosen": 2.0, |
| "approx. KL/rejected": 0.0, |
| "epoch": 0.018442622950819672, |
| "grad_norm": 116.62489318847656, |
| "learning_rate": 1.3877551020408162e-06, |
| "logp/chosen": -1584.0, |
| "logp/rejected": -1440.0, |
| "loss": 1.30322265625, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.125, |
| "reward/chosen": 0.0, |
| "reward/margin": 0.0, |
| "reward/rejected": 0.0, |
| "step": 18 |
| }, |
| { |
| "approx. KL/chosen": 5.25, |
| "approx. KL/rejected": 10.25, |
| "epoch": 0.0194672131147541, |
| "grad_norm": 119.68720245361328, |
| "learning_rate": 1.4693877551020408e-06, |
| "logp/chosen": -968.0, |
| "logp/rejected": -1056.0, |
| "loss": 1.15966796875, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.349609375, |
| "reward/margin": 0.6015625, |
| "reward/rejected": -0.25, |
| "step": 19 |
| }, |
| { |
| "approx. KL/chosen": 1.25, |
| "approx. KL/rejected": 84.0, |
| "epoch": 0.020491803278688523, |
| "grad_norm": 78.77299499511719, |
| "learning_rate": 1.5510204081632651e-06, |
| "logp/chosen": -1200.0, |
| "logp/rejected": -1808.0, |
| "loss": 1.07177734375, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 1.2734375, |
| "reward/rejected": -1.4296875, |
| "step": 20 |
| }, |
| { |
| "approx. KL/chosen": 0.125, |
| "approx. KL/rejected": 20.0, |
| "epoch": 0.02151639344262295, |
| "grad_norm": 74.31775665283203, |
| "learning_rate": 1.6326530612244897e-06, |
| "logp/chosen": -900.0, |
| "logp/rejected": -1112.0, |
| "loss": 1.09814453125, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": 0.0, |
| "reward/margin": 0.6015625, |
| "reward/rejected": -0.6015625, |
| "step": 21 |
| }, |
| { |
| "approx. KL/chosen": 6.0, |
| "approx. KL/rejected": 14.25, |
| "epoch": 0.022540983606557378, |
| "grad_norm": 76.98114013671875, |
| "learning_rate": 1.714285714285714e-06, |
| "logp/chosen": -876.0, |
| "logp/rejected": -1232.0, |
| "loss": 1.1220703125, |
| "nll_loss": 0.66015625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.0, |
| "reward/margin": 0.8515625, |
| "reward/rejected": -0.8515625, |
| "step": 22 |
| }, |
| { |
| "approx. KL/chosen": 0.0, |
| "approx. KL/rejected": 17.0, |
| "epoch": 0.0235655737704918, |
| "grad_norm": 90.6554946899414, |
| "learning_rate": 1.7959183673469386e-06, |
| "logp/chosen": -1144.0, |
| "logp/rejected": -1240.0, |
| "loss": 1.16552734375, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": 0.0, |
| "reward/margin": 0.52734375, |
| "reward/rejected": -0.52734375, |
| "step": 23 |
| }, |
| { |
| "approx. KL/chosen": 0.3125, |
| "approx. KL/rejected": 25.0, |
| "epoch": 0.02459016393442623, |
| "grad_norm": 74.37418365478516, |
| "learning_rate": 1.8775510204081632e-06, |
| "logp/chosen": -1096.0, |
| "logp/rejected": -1696.0, |
| "loss": 0.96875, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.0250244140625, |
| "reward/margin": 0.91796875, |
| "reward/rejected": -0.89453125, |
| "step": 24 |
| }, |
| { |
| "approx. KL/chosen": 18.25, |
| "approx. KL/rejected": 15.25, |
| "epoch": 0.025614754098360656, |
| "grad_norm": 126.29024505615234, |
| "learning_rate": 1.9591836734693877e-06, |
| "logp/chosen": -908.0, |
| "logp/rejected": -1040.0, |
| "loss": 1.24755859375, |
| "nll_loss": 0.45703125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.6640625, |
| "reward/margin": 0.087890625, |
| "reward/rejected": -0.75, |
| "step": 25 |
| }, |
| { |
| "approx. KL/chosen": 6.0, |
| "approx. KL/rejected": 34.25, |
| "epoch": 0.02663934426229508, |
| "grad_norm": 98.2187271118164, |
| "learning_rate": 2.040816326530612e-06, |
| "logp/chosen": -912.0, |
| "logp/rejected": -1008.0, |
| "loss": 1.33984375, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 0.55078125, |
| "reward/rejected": -0.94921875, |
| "step": 26 |
| }, |
| { |
| "approx. KL/chosen": 17.25, |
| "approx. KL/rejected": 29.0, |
| "epoch": 0.027663934426229508, |
| "grad_norm": 105.1951675415039, |
| "learning_rate": 2.122448979591837e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1136.0, |
| "loss": 1.3193359375, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.57421875, |
| "reward/margin": 0.55078125, |
| "reward/rejected": -1.125, |
| "step": 27 |
| }, |
| { |
| "approx. KL/chosen": 6.0, |
| "approx. KL/rejected": 16.125, |
| "epoch": 0.028688524590163935, |
| "grad_norm": 75.95616149902344, |
| "learning_rate": 2.2040816326530608e-06, |
| "logp/chosen": -844.0, |
| "logp/rejected": -972.0, |
| "loss": 1.1552734375, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 0.63671875, |
| "reward/rejected": -0.83984375, |
| "step": 28 |
| }, |
| { |
| "approx. KL/chosen": 5.75, |
| "approx. KL/rejected": 8.1875, |
| "epoch": 0.02971311475409836, |
| "grad_norm": 44.002037048339844, |
| "learning_rate": 2.2857142857142856e-06, |
| "logp/chosen": -568.0, |
| "logp/rejected": -1056.0, |
| "loss": 0.97314453125, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.451171875, |
| "reward/margin": 0.90625, |
| "reward/rejected": -0.45703125, |
| "step": 29 |
| }, |
| { |
| "approx. KL/chosen": 37.0, |
| "approx. KL/rejected": 73.0, |
| "epoch": 0.030737704918032786, |
| "grad_norm": 126.09246063232422, |
| "learning_rate": 2.36734693877551e-06, |
| "logp/chosen": -1360.0, |
| "logp/rejected": -1656.0, |
| "loss": 1.56640625, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 0.6015625, |
| "reward/rejected": -1.703125, |
| "step": 30 |
| }, |
| { |
| "approx. KL/chosen": 1.5, |
| "approx. KL/rejected": 101.0, |
| "epoch": 0.031762295081967214, |
| "grad_norm": 51.62371063232422, |
| "learning_rate": 2.4489795918367347e-06, |
| "logp/chosen": -1136.0, |
| "logp/rejected": -1832.0, |
| "loss": 0.859375, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.10009765625, |
| "reward/margin": 2.015625, |
| "reward/rejected": -1.9140625, |
| "step": 31 |
| }, |
| { |
| "approx. KL/chosen": 16.0, |
| "approx. KL/rejected": 37.0, |
| "epoch": 0.03278688524590164, |
| "grad_norm": 127.31978607177734, |
| "learning_rate": 2.530612244897959e-06, |
| "logp/chosen": -1320.0, |
| "logp/rejected": -1256.0, |
| "loss": 1.353515625, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 0.69921875, |
| "reward/rejected": -1.1015625, |
| "step": 32 |
| }, |
| { |
| "approx. KL/chosen": 4.3125, |
| "approx. KL/rejected": 20.125, |
| "epoch": 0.03381147540983607, |
| "grad_norm": 74.93506622314453, |
| "learning_rate": 2.6122448979591834e-06, |
| "logp/chosen": -776.0, |
| "logp/rejected": -1056.0, |
| "loss": 1.13525390625, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.125, |
| "reward/margin": 0.53125, |
| "reward/rejected": -0.65625, |
| "step": 33 |
| }, |
| { |
| "approx. KL/chosen": 64.0, |
| "approx. KL/rejected": 60.25, |
| "epoch": 0.03483606557377049, |
| "grad_norm": 180.62913513183594, |
| "learning_rate": 2.6938775510204077e-06, |
| "logp/chosen": -1080.0, |
| "logp/rejected": -1240.0, |
| "loss": 1.77978515625, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 0.5, |
| "reward/rejected": -1.3515625, |
| "step": 34 |
| }, |
| { |
| "approx. KL/chosen": 1.5625, |
| "approx. KL/rejected": 41.5, |
| "epoch": 0.035860655737704916, |
| "grad_norm": 59.97579574584961, |
| "learning_rate": 2.7755102040816325e-06, |
| "logp/chosen": -484.0, |
| "logp/rejected": -672.0, |
| "loss": 1.0869140625, |
| "nll_loss": 0.478515625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.2216796875, |
| "reward/margin": 0.67578125, |
| "reward/rejected": -0.8984375, |
| "step": 35 |
| }, |
| { |
| "approx. KL/chosen": 3.3125, |
| "approx. KL/rejected": 33.25, |
| "epoch": 0.036885245901639344, |
| "grad_norm": 70.61190795898438, |
| "learning_rate": 2.8571428571428573e-06, |
| "logp/chosen": -1120.0, |
| "logp/rejected": -996.0, |
| "loss": 1.08984375, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.2255859375, |
| "reward/margin": 0.90234375, |
| "reward/rejected": -1.125, |
| "step": 36 |
| }, |
| { |
| "approx. KL/chosen": 21.0, |
| "approx. KL/rejected": 45.0, |
| "epoch": 0.03790983606557377, |
| "grad_norm": 138.30996704101562, |
| "learning_rate": 2.9387755102040816e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1264.0, |
| "loss": 1.38818359375, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.5, |
| "reward/margin": 0.6015625, |
| "reward/rejected": -1.1015625, |
| "step": 37 |
| }, |
| { |
| "approx. KL/chosen": 9.25, |
| "approx. KL/rejected": 197.0, |
| "epoch": 0.0389344262295082, |
| "grad_norm": 35.17537307739258, |
| "learning_rate": 3.020408163265306e-06, |
| "logp/chosen": -904.0, |
| "logp/rejected": -1696.0, |
| "loss": 0.90869140625, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.25, |
| "reward/margin": 2.921875, |
| "reward/rejected": -2.671875, |
| "step": 38 |
| }, |
| { |
| "approx. KL/chosen": 2.0, |
| "approx. KL/rejected": 22.5, |
| "epoch": 0.039959016393442626, |
| "grad_norm": 50.726966857910156, |
| "learning_rate": 3.1020408163265303e-06, |
| "logp/chosen": -604.0, |
| "logp/rejected": -540.0, |
| "loss": 1.12451171875, |
| "nll_loss": 0.64453125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": 0.10009765625, |
| "reward/margin": 0.80078125, |
| "reward/rejected": -0.69921875, |
| "step": 39 |
| }, |
| { |
| "approx. KL/chosen": 4.0625, |
| "approx. KL/rejected": 45.5, |
| "epoch": 0.040983606557377046, |
| "grad_norm": 84.5871810913086, |
| "learning_rate": 3.183673469387755e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -1168.0, |
| "loss": 1.07470703125, |
| "nll_loss": 0.494140625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.2255859375, |
| "reward/margin": 0.9375, |
| "reward/rejected": -1.1640625, |
| "step": 40 |
| }, |
| { |
| "approx. KL/chosen": 17.5, |
| "approx. KL/rejected": 11.3125, |
| "epoch": 0.042008196721311473, |
| "grad_norm": 140.1292724609375, |
| "learning_rate": 3.2653061224489794e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -868.0, |
| "loss": 1.65283203125, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.400390625, |
| "reward/margin": -0.244140625, |
| "reward/rejected": -0.15625, |
| "step": 41 |
| }, |
| { |
| "approx. KL/chosen": 16.0, |
| "approx. KL/rejected": 43.25, |
| "epoch": 0.0430327868852459, |
| "grad_norm": 123.54158782958984, |
| "learning_rate": 3.346938775510204e-06, |
| "logp/chosen": -1360.0, |
| "logp/rejected": -1112.0, |
| "loss": 1.19091796875, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 0.94921875, |
| "reward/rejected": -1.3515625, |
| "step": 42 |
| }, |
| { |
| "approx. KL/chosen": 3.0625, |
| "approx. KL/rejected": 20.25, |
| "epoch": 0.04405737704918033, |
| "grad_norm": 103.5156478881836, |
| "learning_rate": 3.428571428571428e-06, |
| "logp/chosen": -876.0, |
| "logp/rejected": -1600.0, |
| "loss": 1.2978515625, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.275390625, |
| "reward/margin": 0.375, |
| "reward/rejected": -0.65234375, |
| "step": 43 |
| }, |
| { |
| "approx. KL/chosen": 22.5, |
| "approx. KL/rejected": 23.75, |
| "epoch": 0.045081967213114756, |
| "grad_norm": 122.88953399658203, |
| "learning_rate": 3.510204081632653e-06, |
| "logp/chosen": -1064.0, |
| "logp/rejected": -1032.0, |
| "loss": 1.53271484375, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.69921875, |
| "reward/margin": 0.23046875, |
| "reward/rejected": -0.9296875, |
| "step": 44 |
| }, |
| { |
| "approx. KL/chosen": 6.25, |
| "approx. KL/rejected": 121.5, |
| "epoch": 0.04610655737704918, |
| "grad_norm": 48.85026168823242, |
| "learning_rate": 3.5918367346938772e-06, |
| "logp/chosen": -924.0, |
| "logp/rejected": -1104.0, |
| "loss": 0.98486328125, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.25, |
| "reward/margin": 1.9296875, |
| "reward/rejected": -2.171875, |
| "step": 45 |
| }, |
| { |
| "approx. KL/chosen": 9.625, |
| "approx. KL/rejected": 30.25, |
| "epoch": 0.0471311475409836, |
| "grad_norm": 87.73656463623047, |
| "learning_rate": 3.673469387755102e-06, |
| "logp/chosen": -796.0, |
| "logp/rejected": -892.0, |
| "loss": 1.24658203125, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.451171875, |
| "reward/margin": 0.6015625, |
| "reward/rejected": -1.0546875, |
| "step": 46 |
| }, |
| { |
| "approx. KL/chosen": 23.0, |
| "approx. KL/rejected": 77.0, |
| "epoch": 0.04815573770491803, |
| "grad_norm": 144.7299041748047, |
| "learning_rate": 3.7551020408163264e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1216.0, |
| "loss": 1.60888671875, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.69921875, |
| "reward/margin": 0.80078125, |
| "reward/rejected": -1.5, |
| "step": 47 |
| }, |
| { |
| "approx. KL/chosen": 6.5, |
| "approx. KL/rejected": 25.5, |
| "epoch": 0.04918032786885246, |
| "grad_norm": 81.63092803955078, |
| "learning_rate": 3.83673469387755e-06, |
| "logp/chosen": -832.0, |
| "logp/rejected": -984.0, |
| "loss": 1.25, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.5, |
| "reward/margin": 0.400390625, |
| "reward/rejected": -0.90234375, |
| "step": 48 |
| }, |
| { |
| "approx. KL/chosen": 68.5, |
| "approx. KL/rejected": 59.0, |
| "epoch": 0.050204918032786885, |
| "grad_norm": 97.94182586669922, |
| "learning_rate": 3.9183673469387755e-06, |
| "logp/chosen": -916.0, |
| "logp/rejected": -992.0, |
| "loss": 1.5458984375, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.63671875, |
| "reward/margin": 0.86328125, |
| "reward/rejected": -1.5, |
| "step": 49 |
| }, |
| { |
| "approx. KL/chosen": 1.25, |
| "approx. KL/rejected": 49.25, |
| "epoch": 0.05122950819672131, |
| "grad_norm": 52.72624969482422, |
| "learning_rate": 4e-06, |
| "logp/chosen": -684.0, |
| "logp/rejected": -1088.0, |
| "loss": 1.09521484375, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 1.0234375, |
| "reward/rejected": -1.1796875, |
| "step": 50 |
| }, |
| { |
| "approx. KL/chosen": 16.25, |
| "approx. KL/rejected": 111.0, |
| "epoch": 0.05225409836065574, |
| "grad_norm": 65.92276763916016, |
| "learning_rate": 4.081632653061224e-06, |
| "logp/chosen": -944.0, |
| "logp/rejected": -1192.0, |
| "loss": 1.03759765625, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.451171875, |
| "reward/margin": 1.8984375, |
| "reward/rejected": -2.34375, |
| "step": 51 |
| }, |
| { |
| "approx. KL/chosen": 4.25, |
| "approx. KL/rejected": 36.25, |
| "epoch": 0.05327868852459016, |
| "grad_norm": 80.5989761352539, |
| "learning_rate": 4.163265306122449e-06, |
| "logp/chosen": -1440.0, |
| "logp/rejected": -1296.0, |
| "loss": 0.95556640625, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 1.203125, |
| "reward/rejected": -1.3515625, |
| "step": 52 |
| }, |
| { |
| "approx. KL/chosen": 9.3125, |
| "approx. KL/rejected": 120.5, |
| "epoch": 0.05430327868852459, |
| "grad_norm": 70.23167419433594, |
| "learning_rate": 4.244897959183674e-06, |
| "logp/chosen": -1008.0, |
| "logp/rejected": -1192.0, |
| "loss": 1.06494140625, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.1748046875, |
| "reward/margin": 1.8671875, |
| "reward/rejected": -2.046875, |
| "step": 53 |
| }, |
| { |
| "approx. KL/chosen": 2.25, |
| "approx. KL/rejected": 139.0, |
| "epoch": 0.055327868852459015, |
| "grad_norm": 34.21501159667969, |
| "learning_rate": 4.326530612244898e-06, |
| "logp/chosen": -616.0, |
| "logp/rejected": -964.0, |
| "loss": 0.859375, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.046875, |
| "reward/margin": 2.34375, |
| "reward/rejected": -2.296875, |
| "step": 54 |
| }, |
| { |
| "approx. KL/chosen": 21.25, |
| "approx. KL/rejected": 171.0, |
| "epoch": 0.05635245901639344, |
| "grad_norm": 101.36479949951172, |
| "learning_rate": 4.4081632653061216e-06, |
| "logp/chosen": -924.0, |
| "logp/rejected": -1472.0, |
| "loss": 1.244140625, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 2.125, |
| "reward/rejected": -2.671875, |
| "step": 55 |
| }, |
| { |
| "approx. KL/chosen": 13.25, |
| "approx. KL/rejected": 87.0, |
| "epoch": 0.05737704918032787, |
| "grad_norm": 82.741455078125, |
| "learning_rate": 4.489795918367347e-06, |
| "logp/chosen": -956.0, |
| "logp/rejected": -1160.0, |
| "loss": 1.06298828125, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.25, |
| "reward/margin": 1.4296875, |
| "reward/rejected": -1.6796875, |
| "step": 56 |
| }, |
| { |
| "approx. KL/chosen": 13.5625, |
| "approx. KL/rejected": 44.75, |
| "epoch": 0.0584016393442623, |
| "grad_norm": 118.7864990234375, |
| "learning_rate": 4.571428571428571e-06, |
| "logp/chosen": -724.0, |
| "logp/rejected": -904.0, |
| "loss": 1.56005859375, |
| "nll_loss": 0.490234375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.57421875, |
| "reward/margin": 0.07568359375, |
| "reward/rejected": -0.65234375, |
| "step": 57 |
| }, |
| { |
| "approx. KL/chosen": 11.25, |
| "approx. KL/rejected": 37.0, |
| "epoch": 0.05942622950819672, |
| "grad_norm": 94.95442962646484, |
| "learning_rate": 4.6530612244897954e-06, |
| "logp/chosen": -876.0, |
| "logp/rejected": -1000.0, |
| "loss": 1.380859375, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 0.25, |
| "reward/rejected": -0.90234375, |
| "step": 58 |
| }, |
| { |
| "approx. KL/chosen": 7.0, |
| "approx. KL/rejected": 12.8125, |
| "epoch": 0.060450819672131145, |
| "grad_norm": 79.30979919433594, |
| "learning_rate": 4.73469387755102e-06, |
| "logp/chosen": -784.0, |
| "logp/rejected": -588.0, |
| "loss": 1.24560546875, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.5, |
| "reward/margin": 0.23046875, |
| "reward/rejected": -0.73046875, |
| "step": 59 |
| }, |
| { |
| "approx. KL/chosen": 3.625, |
| "approx. KL/rejected": 57.5, |
| "epoch": 0.06147540983606557, |
| "grad_norm": 44.788509368896484, |
| "learning_rate": 4.816326530612245e-06, |
| "logp/chosen": -624.0, |
| "logp/rejected": -788.0, |
| "loss": 1.1083984375, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 1.1484375, |
| "reward/rejected": -1.3515625, |
| "step": 60 |
| }, |
| { |
| "approx. KL/chosen": 89.0, |
| "approx. KL/rejected": 111.0, |
| "epoch": 0.0625, |
| "grad_norm": 147.98947143554688, |
| "learning_rate": 4.897959183673469e-06, |
| "logp/chosen": -1184.0, |
| "logp/rejected": -1360.0, |
| "loss": 2.77490234375, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.90625, |
| "reward/margin": -0.67578125, |
| "reward/rejected": -1.2265625, |
| "step": 61 |
| }, |
| { |
| "approx. KL/chosen": 9.25, |
| "approx. KL/rejected": 116.5, |
| "epoch": 0.06352459016393443, |
| "grad_norm": 62.26490783691406, |
| "learning_rate": 4.979591836734694e-06, |
| "logp/chosen": -948.0, |
| "logp/rejected": -1416.0, |
| "loss": 0.95849609375, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 1.671875, |
| "reward/rejected": -2.234375, |
| "step": 62 |
| }, |
| { |
| "approx. KL/chosen": 15.0, |
| "approx. KL/rejected": 118.5, |
| "epoch": 0.06454918032786885, |
| "grad_norm": 60.86927032470703, |
| "learning_rate": 5.061224489795918e-06, |
| "logp/chosen": -804.0, |
| "logp/rejected": -1040.0, |
| "loss": 1.23779296875, |
| "nll_loss": 0.68359375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.5, |
| "reward/margin": 1.5625, |
| "reward/rejected": -2.0625, |
| "step": 63 |
| }, |
| { |
| "approx. KL/chosen": 4.3125, |
| "approx. KL/rejected": 42.25, |
| "epoch": 0.06557377049180328, |
| "grad_norm": 54.6102294921875, |
| "learning_rate": 5.142857142857143e-06, |
| "logp/chosen": -932.0, |
| "logp/rejected": -1096.0, |
| "loss": 1.00390625, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.2255859375, |
| "reward/margin": 1.2265625, |
| "reward/rejected": -1.453125, |
| "step": 64 |
| }, |
| { |
| "approx. KL/chosen": 17.25, |
| "approx. KL/rejected": 80.0, |
| "epoch": 0.06659836065573771, |
| "grad_norm": 104.68416595458984, |
| "learning_rate": 5.224489795918367e-06, |
| "logp/chosen": -1004.0, |
| "logp/rejected": -1048.0, |
| "loss": 1.48046875, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.375, |
| "reward/margin": 0.875, |
| "reward/rejected": -1.25, |
| "step": 65 |
| }, |
| { |
| "approx. KL/chosen": 34.0, |
| "approx. KL/rejected": 185.0, |
| "epoch": 0.06762295081967214, |
| "grad_norm": 127.30445098876953, |
| "learning_rate": 5.306122448979591e-06, |
| "logp/chosen": -1168.0, |
| "logp/rejected": -1752.0, |
| "loss": 1.482421875, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.201171875, |
| "reward/margin": 2.296875, |
| "reward/rejected": -2.5, |
| "step": 66 |
| }, |
| { |
| "approx. KL/chosen": 20.25, |
| "approx. KL/rejected": 139.0, |
| "epoch": 0.06864754098360656, |
| "grad_norm": 120.30479431152344, |
| "learning_rate": 5.3877551020408154e-06, |
| "logp/chosen": -1088.0, |
| "logp/rejected": -1768.0, |
| "loss": 1.15869140625, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 1.796875, |
| "reward/rejected": -2.4375, |
| "step": 67 |
| }, |
| { |
| "approx. KL/chosen": 11.0625, |
| "approx. KL/rejected": 188.0, |
| "epoch": 0.06967213114754098, |
| "grad_norm": 66.28058624267578, |
| "learning_rate": 5.469387755102041e-06, |
| "logp/chosen": -700.0, |
| "logp/rejected": -1080.0, |
| "loss": 1.1689453125, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.0751953125, |
| "reward/margin": 2.21875, |
| "reward/rejected": -2.296875, |
| "step": 68 |
| }, |
| { |
| "approx. KL/chosen": 5.3125, |
| "approx. KL/rejected": 136.0, |
| "epoch": 0.0706967213114754, |
| "grad_norm": 25.90762710571289, |
| "learning_rate": 5.551020408163265e-06, |
| "logp/chosen": -1004.0, |
| "logp/rejected": -1600.0, |
| "loss": 0.74609375, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.375, |
| "reward/margin": 2.453125, |
| "reward/rejected": -2.828125, |
| "step": 69 |
| }, |
| { |
| "approx. KL/chosen": 28.5, |
| "approx. KL/rejected": 7.0625, |
| "epoch": 0.07172131147540983, |
| "grad_norm": 159.36756896972656, |
| "learning_rate": 5.632653061224489e-06, |
| "logp/chosen": -1112.0, |
| "logp/rejected": -724.0, |
| "loss": 1.6748046875, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -0.90234375, |
| "reward/margin": -0.326171875, |
| "reward/rejected": -0.57421875, |
| "step": 70 |
| }, |
| { |
| "approx. KL/chosen": 24.25, |
| "approx. KL/rejected": 165.0, |
| "epoch": 0.07274590163934426, |
| "grad_norm": 133.58290100097656, |
| "learning_rate": 5.7142857142857145e-06, |
| "logp/chosen": -1320.0, |
| "logp/rejected": -1408.0, |
| "loss": 1.34716796875, |
| "nll_loss": 0.5, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.337890625, |
| "reward/margin": 1.59375, |
| "reward/rejected": -1.9296875, |
| "step": 71 |
| }, |
| { |
| "approx. KL/chosen": 11.0625, |
| "approx. KL/rejected": 108.0, |
| "epoch": 0.07377049180327869, |
| "grad_norm": 74.83979034423828, |
| "learning_rate": 5.795918367346939e-06, |
| "logp/chosen": -956.0, |
| "logp/rejected": -1184.0, |
| "loss": 1.07275390625, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.326171875, |
| "reward/margin": 1.7265625, |
| "reward/rejected": -1.40625, |
| "step": 72 |
| }, |
| { |
| "approx. KL/chosen": 5.5625, |
| "approx. KL/rejected": 49.0, |
| "epoch": 0.07479508196721311, |
| "grad_norm": 34.04027557373047, |
| "learning_rate": 5.877551020408163e-06, |
| "logp/chosen": -664.0, |
| "logp/rejected": -912.0, |
| "loss": 0.9638671875, |
| "nll_loss": 0.640625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.2255859375, |
| "reward/margin": 1.3984375, |
| "reward/rejected": -1.625, |
| "step": 73 |
| }, |
| { |
| "approx. KL/chosen": 7.0625, |
| "approx. KL/rejected": 140.0, |
| "epoch": 0.07581967213114754, |
| "grad_norm": 56.39397048950195, |
| "learning_rate": 5.9591836734693876e-06, |
| "logp/chosen": -760.0, |
| "logp/rejected": -1152.0, |
| "loss": 0.958984375, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.125, |
| "reward/margin": 2.328125, |
| "reward/rejected": -2.203125, |
| "step": 74 |
| }, |
| { |
| "approx. KL/chosen": 20.25, |
| "approx. KL/rejected": 185.0, |
| "epoch": 0.07684426229508197, |
| "grad_norm": 91.72496795654297, |
| "learning_rate": 6.040816326530612e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -1152.0, |
| "loss": 1.158203125, |
| "nll_loss": 0.515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 2.015625, |
| "reward/rejected": -2.5625, |
| "step": 75 |
| }, |
| { |
| "approx. KL/chosen": 0.25, |
| "approx. KL/rejected": 84.5, |
| "epoch": 0.0778688524590164, |
| "grad_norm": 80.39318084716797, |
| "learning_rate": 6.122448979591836e-06, |
| "logp/chosen": -1352.0, |
| "logp/rejected": -1424.0, |
| "loss": 0.84619140625, |
| "nll_loss": 0.443359375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.050048828125, |
| "reward/margin": 1.5625, |
| "reward/rejected": -1.515625, |
| "step": 76 |
| }, |
| { |
| "approx. KL/chosen": 5.5625, |
| "approx. KL/rejected": 40.0, |
| "epoch": 0.07889344262295082, |
| "grad_norm": 123.58467864990234, |
| "learning_rate": 6.204081632653061e-06, |
| "logp/chosen": -1272.0, |
| "logp/rejected": -1600.0, |
| "loss": 1.279296875, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.224609375, |
| "reward/margin": 0.578125, |
| "reward/rejected": -0.80078125, |
| "step": 77 |
| }, |
| { |
| "approx. KL/chosen": 17.5, |
| "approx. KL/rejected": 89.0, |
| "epoch": 0.07991803278688525, |
| "grad_norm": 83.1110610961914, |
| "learning_rate": 6.285714285714285e-06, |
| "logp/chosen": -912.0, |
| "logp/rejected": -1168.0, |
| "loss": 1.16943359375, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.5, |
| "reward/margin": 1.3515625, |
| "reward/rejected": -1.8515625, |
| "step": 78 |
| }, |
| { |
| "approx. KL/chosen": 9.5, |
| "approx. KL/rejected": 79.5, |
| "epoch": 0.08094262295081968, |
| "grad_norm": 38.32463073730469, |
| "learning_rate": 6.36734693877551e-06, |
| "logp/chosen": -908.0, |
| "logp/rejected": -1160.0, |
| "loss": 0.8310546875, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.400390625, |
| "reward/margin": 2.078125, |
| "reward/rejected": -1.6796875, |
| "step": 79 |
| }, |
| { |
| "approx. KL/chosen": 11.0, |
| "approx. KL/rejected": 77.0, |
| "epoch": 0.08196721311475409, |
| "grad_norm": 65.594482421875, |
| "learning_rate": 6.4489795918367345e-06, |
| "logp/chosen": -1096.0, |
| "logp/rejected": -1568.0, |
| "loss": 0.87060546875, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.10009765625, |
| "reward/margin": 2.0, |
| "reward/rejected": -1.90625, |
| "step": 80 |
| }, |
| { |
| "approx. KL/chosen": 9.3125, |
| "approx. KL/rejected": 135.0, |
| "epoch": 0.08299180327868852, |
| "grad_norm": 33.83040237426758, |
| "learning_rate": 6.530612244897959e-06, |
| "logp/chosen": -748.0, |
| "logp/rejected": -1256.0, |
| "loss": 0.85009765625, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.0751953125, |
| "reward/margin": 2.53125, |
| "reward/rejected": -2.453125, |
| "step": 81 |
| }, |
| { |
| "approx. KL/chosen": 1.640625, |
| "approx. KL/rejected": 36.0, |
| "epoch": 0.08401639344262295, |
| "grad_norm": 54.63899612426758, |
| "learning_rate": 6.612244897959184e-06, |
| "logp/chosen": -652.0, |
| "logp/rejected": -916.0, |
| "loss": 0.997802734375, |
| "nll_loss": 0.4921875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.1376953125, |
| "reward/margin": 0.875, |
| "reward/rejected": -1.015625, |
| "step": 82 |
| }, |
| { |
| "approx. KL/chosen": 3.0, |
| "approx. KL/rejected": 211.0, |
| "epoch": 0.08504098360655737, |
| "grad_norm": 25.503734588623047, |
| "learning_rate": 6.693877551020408e-06, |
| "logp/chosen": -968.0, |
| "logp/rejected": -1512.0, |
| "loss": 0.71337890625, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 3.328125, |
| "reward/rejected": -3.421875, |
| "step": 83 |
| }, |
| { |
| "approx. KL/chosen": 152.0, |
| "approx. KL/rejected": 58.25, |
| "epoch": 0.0860655737704918, |
| "grad_norm": 133.80145263671875, |
| "learning_rate": 6.775510204081633e-06, |
| "logp/chosen": -1008.0, |
| "logp/rejected": -860.0, |
| "loss": 2.3828125, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.6015625, |
| "reward/margin": -0.349609375, |
| "reward/rejected": -1.25, |
| "step": 84 |
| }, |
| { |
| "approx. KL/chosen": 7.625, |
| "approx. KL/rejected": 33.75, |
| "epoch": 0.08709016393442623, |
| "grad_norm": 59.318565368652344, |
| "learning_rate": 6.857142857142856e-06, |
| "logp/chosen": -680.0, |
| "logp/rejected": -688.0, |
| "loss": 1.09716796875, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.349609375, |
| "reward/margin": 0.76953125, |
| "reward/rejected": -1.1171875, |
| "step": 85 |
| }, |
| { |
| "approx. KL/chosen": 4.25, |
| "approx. KL/rejected": 123.0, |
| "epoch": 0.08811475409836066, |
| "grad_norm": 22.12877082824707, |
| "learning_rate": 6.938775510204081e-06, |
| "logp/chosen": -764.0, |
| "logp/rejected": -1272.0, |
| "loss": 0.64111328125, |
| "nll_loss": 0.482421875, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.050048828125, |
| "reward/margin": 2.5, |
| "reward/rejected": -2.546875, |
| "step": 86 |
| }, |
| { |
| "approx. KL/chosen": 79.0, |
| "approx. KL/rejected": 72.0, |
| "epoch": 0.08913934426229508, |
| "grad_norm": 144.30857849121094, |
| "learning_rate": 7.020408163265306e-06, |
| "logp/chosen": -1264.0, |
| "logp/rejected": -1224.0, |
| "loss": 1.81640625, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.7265625, |
| "reward/margin": 0.828125, |
| "reward/rejected": -1.5546875, |
| "step": 87 |
| }, |
| { |
| "approx. KL/chosen": 45.5, |
| "approx. KL/rejected": 85.5, |
| "epoch": 0.09016393442622951, |
| "grad_norm": 92.30307006835938, |
| "learning_rate": 7.10204081632653e-06, |
| "logp/chosen": -572.0, |
| "logp/rejected": -568.0, |
| "loss": 1.77490234375, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 0.435546875, |
| "reward/rejected": -1.5390625, |
| "step": 88 |
| }, |
| { |
| "approx. KL/chosen": 33.0, |
| "approx. KL/rejected": 156.0, |
| "epoch": 0.09118852459016394, |
| "grad_norm": 46.161006927490234, |
| "learning_rate": 7.1836734693877545e-06, |
| "logp/chosen": -1480.0, |
| "logp/rejected": -1928.0, |
| "loss": 0.7216796875, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.5, |
| "reward/margin": 3.125, |
| "reward/rejected": -2.625, |
| "step": 89 |
| }, |
| { |
| "approx. KL/chosen": 11.0625, |
| "approx. KL/rejected": 128.0, |
| "epoch": 0.09221311475409837, |
| "grad_norm": 65.38282775878906, |
| "learning_rate": 7.26530612244898e-06, |
| "logp/chosen": -640.0, |
| "logp/rejected": -1020.0, |
| "loss": 1.23828125, |
| "nll_loss": 0.6484375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.52734375, |
| "reward/margin": 1.65625, |
| "reward/rejected": -2.171875, |
| "step": 90 |
| }, |
| { |
| "approx. KL/chosen": 18.25, |
| "approx. KL/rejected": 224.0, |
| "epoch": 0.0932377049180328, |
| "grad_norm": 44.76226806640625, |
| "learning_rate": 7.346938775510204e-06, |
| "logp/chosen": -920.0, |
| "logp/rejected": -1096.0, |
| "loss": 0.9755859375, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.3515625, |
| "reward/margin": 2.859375, |
| "reward/rejected": -3.203125, |
| "step": 91 |
| }, |
| { |
| "approx. KL/chosen": 38.0, |
| "approx. KL/rejected": 125.5, |
| "epoch": 0.0942622950819672, |
| "grad_norm": 130.95457458496094, |
| "learning_rate": 7.428571428571428e-06, |
| "logp/chosen": -1160.0, |
| "logp/rejected": -1208.0, |
| "loss": 1.68017578125, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.4296875, |
| "reward/margin": 0.7421875, |
| "reward/rejected": -2.171875, |
| "step": 92 |
| }, |
| { |
| "approx. KL/chosen": 10.5625, |
| "approx. KL/rejected": 136.0, |
| "epoch": 0.09528688524590163, |
| "grad_norm": 25.127424240112305, |
| "learning_rate": 7.510204081632653e-06, |
| "logp/chosen": -736.0, |
| "logp/rejected": -1224.0, |
| "loss": 0.72412109375, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.17578125, |
| "reward/margin": 2.828125, |
| "reward/rejected": -2.65625, |
| "step": 93 |
| }, |
| { |
| "approx. KL/chosen": 15.0, |
| "approx. KL/rejected": 364.0, |
| "epoch": 0.09631147540983606, |
| "grad_norm": 74.17122650146484, |
| "learning_rate": 7.591836734693878e-06, |
| "logp/chosen": -1016.0, |
| "logp/rejected": -1416.0, |
| "loss": 1.0029296875, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.10009765625, |
| "reward/margin": 4.21875, |
| "reward/rejected": -4.09375, |
| "step": 94 |
| }, |
| { |
| "approx. KL/chosen": 6.625, |
| "approx. KL/rejected": 251.0, |
| "epoch": 0.09733606557377049, |
| "grad_norm": 36.47000503540039, |
| "learning_rate": 7.6734693877551e-06, |
| "logp/chosen": -820.0, |
| "logp/rejected": -1376.0, |
| "loss": 0.84130859375, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.0498046875, |
| "reward/margin": 3.375, |
| "reward/rejected": -3.421875, |
| "step": 95 |
| }, |
| { |
| "approx. KL/chosen": 18.0, |
| "approx. KL/rejected": 55.75, |
| "epoch": 0.09836065573770492, |
| "grad_norm": 81.03041076660156, |
| "learning_rate": 7.755102040816326e-06, |
| "logp/chosen": -904.0, |
| "logp/rejected": -1128.0, |
| "loss": 1.02294921875, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.61328125, |
| "reward/margin": 1.109375, |
| "reward/rejected": -1.7265625, |
| "step": 96 |
| }, |
| { |
| "approx. KL/chosen": 29.125, |
| "approx. KL/rejected": 69.0, |
| "epoch": 0.09938524590163934, |
| "grad_norm": 102.95508575439453, |
| "learning_rate": 7.836734693877551e-06, |
| "logp/chosen": -952.0, |
| "logp/rejected": -1168.0, |
| "loss": 1.56298828125, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.5390625, |
| "reward/margin": 1.0, |
| "reward/rejected": -1.5390625, |
| "step": 97 |
| }, |
| { |
| "approx. KL/chosen": 37.0, |
| "approx. KL/rejected": 225.0, |
| "epoch": 0.10040983606557377, |
| "grad_norm": 114.86624145507812, |
| "learning_rate": 7.918367346938774e-06, |
| "logp/chosen": -1384.0, |
| "logp/rejected": -1552.0, |
| "loss": 1.14501953125, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.69921875, |
| "reward/margin": 2.0, |
| "reward/rejected": -2.703125, |
| "step": 98 |
| }, |
| { |
| "approx. KL/chosen": 89.0, |
| "approx. KL/rejected": 223.0, |
| "epoch": 0.1014344262295082, |
| "grad_norm": 122.1392822265625, |
| "learning_rate": 8e-06, |
| "logp/chosen": -1176.0, |
| "logp/rejected": -1272.0, |
| "loss": 1.787109375, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.3046875, |
| "reward/margin": 1.59375, |
| "reward/rejected": -2.890625, |
| "step": 99 |
| }, |
| { |
| "approx. KL/chosen": 188.0, |
| "approx. KL/rejected": 32.25, |
| "epoch": 0.10245901639344263, |
| "grad_norm": 183.84645080566406, |
| "learning_rate": 7.999976954679345e-06, |
| "logp/chosen": -1336.0, |
| "logp/rejected": -1112.0, |
| "loss": 2.791015625, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -2.15625, |
| "reward/margin": -1.3046875, |
| "reward/rejected": -0.8515625, |
| "step": 100 |
| }, |
| { |
| "approx. KL/chosen": 6.0625, |
| "approx. KL/rejected": 294.0, |
| "epoch": 0.10348360655737705, |
| "grad_norm": 20.501819610595703, |
| "learning_rate": 7.999907819012427e-06, |
| "logp/chosen": -1064.0, |
| "logp/rejected": -1752.0, |
| "loss": 0.72265625, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.07470703125, |
| "reward/margin": 4.25, |
| "reward/rejected": -4.34375, |
| "step": 101 |
| }, |
| { |
| "approx. KL/chosen": 27.0, |
| "approx. KL/rejected": 46.5, |
| "epoch": 0.10450819672131148, |
| "grad_norm": 139.40753173828125, |
| "learning_rate": 7.999792593884389e-06, |
| "logp/chosen": -1080.0, |
| "logp/rejected": -1160.0, |
| "loss": 1.4365234375, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.703125, |
| "reward/margin": 0.69921875, |
| "reward/rejected": -1.3984375, |
| "step": 102 |
| }, |
| { |
| "approx. KL/chosen": 29.25, |
| "approx. KL/rejected": 54.0, |
| "epoch": 0.10553278688524591, |
| "grad_norm": 111.40092468261719, |
| "learning_rate": 7.999631280770451e-06, |
| "logp/chosen": -976.0, |
| "logp/rejected": -944.0, |
| "loss": 1.7412109375, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.349609375, |
| "reward/margin": 0.376953125, |
| "reward/rejected": -0.7265625, |
| "step": 103 |
| }, |
| { |
| "approx. KL/chosen": 15.5, |
| "approx. KL/rejected": 149.0, |
| "epoch": 0.10655737704918032, |
| "grad_norm": 92.7018051147461, |
| "learning_rate": 7.9994238817359e-06, |
| "logp/chosen": -976.0, |
| "logp/rejected": -1208.0, |
| "loss": 1.29345703125, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 1.65625, |
| "reward/rejected": -2.25, |
| "step": 104 |
| }, |
| { |
| "approx. KL/chosen": 30.0, |
| "approx. KL/rejected": 21.0, |
| "epoch": 0.10758196721311475, |
| "grad_norm": 80.57099151611328, |
| "learning_rate": 7.999170399436055e-06, |
| "logp/chosen": -692.0, |
| "logp/rejected": -1000.0, |
| "loss": 1.22509765625, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.19921875, |
| "reward/margin": 0.703125, |
| "reward/rejected": -0.90234375, |
| "step": 105 |
| }, |
| { |
| "approx. KL/chosen": 30.5, |
| "approx. KL/rejected": 110.0, |
| "epoch": 0.10860655737704918, |
| "grad_norm": 123.25397491455078, |
| "learning_rate": 7.998870837116238e-06, |
| "logp/chosen": -892.0, |
| "logp/rejected": -864.0, |
| "loss": 1.65625, |
| "nll_loss": 0.66796875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.69921875, |
| "reward/margin": 0.6328125, |
| "reward/rejected": -1.328125, |
| "step": 106 |
| }, |
| { |
| "approx. KL/chosen": 27.25, |
| "approx. KL/rejected": 96.0, |
| "epoch": 0.1096311475409836, |
| "grad_norm": 99.1786880493164, |
| "learning_rate": 7.998525198611735e-06, |
| "logp/chosen": -1136.0, |
| "logp/rejected": -1344.0, |
| "loss": 1.5302734375, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.6484375, |
| "reward/margin": 0.90234375, |
| "reward/rejected": -1.5546875, |
| "step": 107 |
| }, |
| { |
| "approx. KL/chosen": 26.5, |
| "approx. KL/rejected": 100.0, |
| "epoch": 0.11065573770491803, |
| "grad_norm": 104.4559097290039, |
| "learning_rate": 7.998133488347738e-06, |
| "logp/chosen": -800.0, |
| "logp/rejected": -1296.0, |
| "loss": 1.22509765625, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.88671875, |
| "reward/margin": 0.96484375, |
| "reward/rejected": -1.8515625, |
| "step": 108 |
| }, |
| { |
| "approx. KL/chosen": 6.0625, |
| "approx. KL/rejected": 29.75, |
| "epoch": 0.11168032786885246, |
| "grad_norm": 77.3673095703125, |
| "learning_rate": 7.997695711339297e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1080.0, |
| "loss": 1.2724609375, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.42578125, |
| "reward/margin": 0.5390625, |
| "reward/rejected": -0.96484375, |
| "step": 109 |
| }, |
| { |
| "approx. KL/chosen": 81.0, |
| "approx. KL/rejected": 318.0, |
| "epoch": 0.11270491803278689, |
| "grad_norm": 114.27873992919922, |
| "learning_rate": 7.997211873191251e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1352.0, |
| "loss": 1.68017578125, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.3515625, |
| "reward/margin": 2.40625, |
| "reward/rejected": -3.75, |
| "step": 110 |
| }, |
| { |
| "approx. KL/chosen": 6.0625, |
| "approx. KL/rejected": 205.0, |
| "epoch": 0.11372950819672131, |
| "grad_norm": 72.8662109375, |
| "learning_rate": 7.996681980098161e-06, |
| "logp/chosen": -968.0, |
| "logp/rejected": -1552.0, |
| "loss": 1.11572265625, |
| "nll_loss": 0.64453125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.375, |
| "reward/margin": 2.328125, |
| "reward/rejected": -2.703125, |
| "step": 111 |
| }, |
| { |
| "approx. KL/chosen": 69.0, |
| "approx. KL/rejected": 186.0, |
| "epoch": 0.11475409836065574, |
| "grad_norm": 194.6909637451172, |
| "learning_rate": 7.996106038844222e-06, |
| "logp/chosen": -1736.0, |
| "logp/rejected": -1984.0, |
| "loss": 1.02490234375, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 2.5, |
| "reward/rejected": -3.359375, |
| "step": 112 |
| }, |
| { |
| "approx. KL/chosen": 14.25, |
| "approx. KL/rejected": 139.0, |
| "epoch": 0.11577868852459017, |
| "grad_norm": 45.827545166015625, |
| "learning_rate": 7.995484056803188e-06, |
| "logp/chosen": -824.0, |
| "logp/rejected": -1176.0, |
| "loss": 1.04345703125, |
| "nll_loss": 0.64453125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 2.3125, |
| "reward/rejected": -2.859375, |
| "step": 113 |
| }, |
| { |
| "approx. KL/chosen": 6.5625, |
| "approx. KL/rejected": 76.5, |
| "epoch": 0.1168032786885246, |
| "grad_norm": 58.31374740600586, |
| "learning_rate": 7.994816041938262e-06, |
| "logp/chosen": -892.0, |
| "logp/rejected": -1168.0, |
| "loss": 0.87353515625, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.419921875, |
| "reward/margin": 1.484375, |
| "reward/rejected": -1.90625, |
| "step": 114 |
| }, |
| { |
| "approx. KL/chosen": 9.0, |
| "approx. KL/rejected": 133.0, |
| "epoch": 0.11782786885245902, |
| "grad_norm": 71.65962982177734, |
| "learning_rate": 7.994102002802013e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1240.0, |
| "loss": 1.01220703125, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 2.25, |
| "reward/rejected": -2.546875, |
| "step": 115 |
| }, |
| { |
| "approx. KL/chosen": 30.0, |
| "approx. KL/rejected": 142.0, |
| "epoch": 0.11885245901639344, |
| "grad_norm": 61.07149124145508, |
| "learning_rate": 7.993341948536253e-06, |
| "logp/chosen": -704.0, |
| "logp/rejected": -888.0, |
| "loss": 1.0703125, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.203125, |
| "reward/margin": 1.296875, |
| "reward/rejected": -2.5, |
| "step": 116 |
| }, |
| { |
| "approx. KL/chosen": 54.0, |
| "approx. KL/rejected": 134.0, |
| "epoch": 0.11987704918032786, |
| "grad_norm": 93.54722595214844, |
| "learning_rate": 7.992535888871922e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1608.0, |
| "loss": 1.53515625, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.6015625, |
| "reward/margin": 1.0, |
| "reward/rejected": -2.609375, |
| "step": 117 |
| }, |
| { |
| "approx. KL/chosen": 23.5, |
| "approx. KL/rejected": 77.5, |
| "epoch": 0.12090163934426229, |
| "grad_norm": 124.8772964477539, |
| "learning_rate": 7.991683834128966e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1344.0, |
| "loss": 1.53271484375, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 0.63671875, |
| "reward/rejected": -1.5390625, |
| "step": 118 |
| }, |
| { |
| "approx. KL/chosen": 16.0, |
| "approx. KL/rejected": 137.0, |
| "epoch": 0.12192622950819672, |
| "grad_norm": 52.67527389526367, |
| "learning_rate": 7.990785795216204e-06, |
| "logp/chosen": -652.0, |
| "logp/rejected": -1528.0, |
| "loss": 0.941650390625, |
| "nll_loss": 0.5078125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.42578125, |
| "reward/margin": 2.109375, |
| "reward/rejected": -2.53125, |
| "step": 119 |
| }, |
| { |
| "approx. KL/chosen": 19.25, |
| "approx. KL/rejected": 88.0, |
| "epoch": 0.12295081967213115, |
| "grad_norm": 98.98299407958984, |
| "learning_rate": 7.989841783631191e-06, |
| "logp/chosen": -848.0, |
| "logp/rejected": -1176.0, |
| "loss": 1.095703125, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.57421875, |
| "reward/margin": 1.2265625, |
| "reward/rejected": -1.8046875, |
| "step": 120 |
| }, |
| { |
| "approx. KL/chosen": 19.375, |
| "approx. KL/rejected": 63.25, |
| "epoch": 0.12397540983606557, |
| "grad_norm": 90.94795989990234, |
| "learning_rate": 7.988851811460062e-06, |
| "logp/chosen": -708.0, |
| "logp/rejected": -1024.0, |
| "loss": 0.981689453125, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.640625, |
| "reward/margin": 1.1484375, |
| "reward/rejected": -1.7890625, |
| "step": 121 |
| }, |
| { |
| "approx. KL/chosen": 8.25, |
| "approx. KL/rejected": 142.0, |
| "epoch": 0.125, |
| "grad_norm": 83.96864318847656, |
| "learning_rate": 7.987815891377391e-06, |
| "logp/chosen": -1152.0, |
| "logp/rejected": -1568.0, |
| "loss": 1.034912109375, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.050048828125, |
| "reward/margin": 2.703125, |
| "reward/rejected": -2.65625, |
| "step": 122 |
| }, |
| { |
| "approx. KL/chosen": 57.25, |
| "approx. KL/rejected": 92.0, |
| "epoch": 0.1260245901639344, |
| "grad_norm": 105.90513610839844, |
| "learning_rate": 7.986734036646015e-06, |
| "logp/chosen": -1096.0, |
| "logp/rejected": -1208.0, |
| "loss": 1.47412109375, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.65625, |
| "reward/margin": 0.703125, |
| "reward/rejected": -2.359375, |
| "step": 123 |
| }, |
| { |
| "approx. KL/chosen": 26.25, |
| "approx. KL/rejected": 10.875, |
| "epoch": 0.12704918032786885, |
| "grad_norm": 87.67095184326172, |
| "learning_rate": 7.985606261116873e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -912.0, |
| "loss": 1.158203125, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": 0.050048828125, |
| "reward/margin": 0.6015625, |
| "reward/rejected": -0.55078125, |
| "step": 124 |
| }, |
| { |
| "approx. KL/chosen": 10.125, |
| "approx. KL/rejected": 16.25, |
| "epoch": 0.12807377049180327, |
| "grad_norm": 62.057464599609375, |
| "learning_rate": 7.984432579228824e-06, |
| "logp/chosen": -568.0, |
| "logp/rejected": -588.0, |
| "loss": 1.17041015625, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.349609375, |
| "reward/margin": 0.54296875, |
| "reward/rejected": -0.89453125, |
| "step": 125 |
| }, |
| { |
| "approx. KL/chosen": 9.0, |
| "approx. KL/rejected": 157.0, |
| "epoch": 0.1290983606557377, |
| "grad_norm": 56.210208892822266, |
| "learning_rate": 7.983213006008468e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1952.0, |
| "loss": 0.859375, |
| "nll_loss": 0.65625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.5, |
| "reward/margin": 2.609375, |
| "reward/rejected": -3.109375, |
| "step": 126 |
| }, |
| { |
| "approx. KL/chosen": 8.0, |
| "approx. KL/rejected": 48.5, |
| "epoch": 0.13012295081967212, |
| "grad_norm": 47.91559982299805, |
| "learning_rate": 7.981947557069945e-06, |
| "logp/chosen": -620.0, |
| "logp/rejected": -1032.0, |
| "loss": 1.021484375, |
| "nll_loss": 0.6796875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 1.5, |
| "reward/rejected": -1.703125, |
| "step": 127 |
| }, |
| { |
| "approx. KL/chosen": 7.5, |
| "approx. KL/rejected": 17.0, |
| "epoch": 0.13114754098360656, |
| "grad_norm": 87.50953674316406, |
| "learning_rate": 7.980636248614742e-06, |
| "logp/chosen": -748.0, |
| "logp/rejected": -1012.0, |
| "loss": 1.28466796875, |
| "nll_loss": 0.498046875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.3125, |
| "reward/margin": 0.2060546875, |
| "reward/rejected": -0.51953125, |
| "step": 128 |
| }, |
| { |
| "approx. KL/chosen": 6.0, |
| "approx. KL/rejected": 68.0, |
| "epoch": 0.13217213114754098, |
| "grad_norm": 58.78346633911133, |
| "learning_rate": 7.979279097431485e-06, |
| "logp/chosen": -1256.0, |
| "logp/rejected": -1680.0, |
| "loss": 0.8818359375, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.2001953125, |
| "reward/margin": 2.0, |
| "reward/rejected": -1.8046875, |
| "step": 129 |
| }, |
| { |
| "approx. KL/chosen": 18.25, |
| "approx. KL/rejected": 48.0, |
| "epoch": 0.13319672131147542, |
| "grad_norm": 100.9955062866211, |
| "learning_rate": 7.97787612089572e-06, |
| "logp/chosen": -740.0, |
| "logp/rejected": -852.0, |
| "loss": 1.51220703125, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 0.125, |
| "reward/rejected": -0.9765625, |
| "step": 130 |
| }, |
| { |
| "approx. KL/chosen": 27.25, |
| "approx. KL/rejected": 43.0, |
| "epoch": 0.13422131147540983, |
| "grad_norm": 100.97091674804688, |
| "learning_rate": 7.976427336969692e-06, |
| "logp/chosen": -908.0, |
| "logp/rejected": -1240.0, |
| "loss": 1.57861328125, |
| "nll_loss": 0.6484375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.451171875, |
| "reward/margin": 0.65234375, |
| "reward/rejected": -1.1015625, |
| "step": 131 |
| }, |
| { |
| "approx. KL/chosen": 27.25, |
| "approx. KL/rejected": 320.0, |
| "epoch": 0.13524590163934427, |
| "grad_norm": 135.17086791992188, |
| "learning_rate": 7.974932764202122e-06, |
| "logp/chosen": -816.0, |
| "logp/rejected": -1512.0, |
| "loss": 1.32958984375, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.94921875, |
| "reward/margin": 2.546875, |
| "reward/rejected": -3.5, |
| "step": 132 |
| }, |
| { |
| "approx. KL/chosen": 22.375, |
| "approx. KL/rejected": 100.5, |
| "epoch": 0.1362704918032787, |
| "grad_norm": 42.43196487426758, |
| "learning_rate": 7.973392421727955e-06, |
| "logp/chosen": -828.0, |
| "logp/rejected": -960.0, |
| "loss": 0.794921875, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.138671875, |
| "reward/margin": 2.625, |
| "reward/rejected": -2.484375, |
| "step": 133 |
| }, |
| { |
| "approx. KL/chosen": 27.0, |
| "approx. KL/rejected": 35.5, |
| "epoch": 0.13729508196721313, |
| "grad_norm": 116.10775756835938, |
| "learning_rate": 7.97180632926813e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -764.0, |
| "loss": 1.56005859375, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.1015625, |
| "reward/margin": -0.06201171875, |
| "reward/rejected": -1.0390625, |
| "step": 134 |
| }, |
| { |
| "approx. KL/chosen": 33.75, |
| "approx. KL/rejected": 288.0, |
| "epoch": 0.13831967213114754, |
| "grad_norm": 40.50993347167969, |
| "learning_rate": 7.970174507129318e-06, |
| "logp/chosen": -1144.0, |
| "logp/rejected": -1904.0, |
| "loss": 0.98876953125, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -1.1875, |
| "reward/margin": 2.828125, |
| "reward/rejected": -4.03125, |
| "step": 135 |
| }, |
| { |
| "approx. KL/chosen": 153.0, |
| "approx. KL/rejected": 99.5, |
| "epoch": 0.13934426229508196, |
| "grad_norm": 176.31381225585938, |
| "learning_rate": 7.968496976203667e-06, |
| "logp/chosen": -904.0, |
| "logp/rejected": -872.0, |
| "loss": 2.18408203125, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.75, |
| "reward/margin": 0.2109375, |
| "reward/rejected": -1.96875, |
| "step": 136 |
| }, |
| { |
| "approx. KL/chosen": 12.0625, |
| "approx. KL/rejected": 112.5, |
| "epoch": 0.1403688524590164, |
| "grad_norm": 48.97671127319336, |
| "learning_rate": 7.966773757968528e-06, |
| "logp/chosen": -952.0, |
| "logp/rejected": -984.0, |
| "loss": 0.92919921875, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.625, |
| "reward/margin": 1.8359375, |
| "reward/rejected": -2.453125, |
| "step": 137 |
| }, |
| { |
| "approx. KL/chosen": 20.25, |
| "approx. KL/rejected": 66.0, |
| "epoch": 0.1413934426229508, |
| "grad_norm": 66.998046875, |
| "learning_rate": 7.965004874486193e-06, |
| "logp/chosen": -520.0, |
| "logp/rejected": -576.0, |
| "loss": 1.1337890625, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.474609375, |
| "reward/margin": 1.2734375, |
| "reward/rejected": -1.75, |
| "step": 138 |
| }, |
| { |
| "approx. KL/chosen": 77.0, |
| "approx. KL/rejected": 82.5, |
| "epoch": 0.14241803278688525, |
| "grad_norm": 138.04098510742188, |
| "learning_rate": 7.963190348403596e-06, |
| "logp/chosen": -1120.0, |
| "logp/rejected": -1104.0, |
| "loss": 1.509765625, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.5, |
| "reward/margin": 0.4296875, |
| "reward/rejected": -1.9296875, |
| "step": 139 |
| }, |
| { |
| "approx. KL/chosen": 13.25, |
| "approx. KL/rejected": 218.0, |
| "epoch": 0.14344262295081966, |
| "grad_norm": 30.88227653503418, |
| "learning_rate": 7.961330202952037e-06, |
| "logp/chosen": -884.0, |
| "logp/rejected": -1552.0, |
| "loss": 0.7763671875, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.150390625, |
| "reward/margin": 3.546875, |
| "reward/rejected": -3.40625, |
| "step": 140 |
| }, |
| { |
| "approx. KL/chosen": 23.5, |
| "approx. KL/rejected": 222.0, |
| "epoch": 0.1444672131147541, |
| "grad_norm": 289.960693359375, |
| "learning_rate": 7.959424461946873e-06, |
| "logp/chosen": -948.0, |
| "logp/rejected": -1336.0, |
| "loss": 1.2744140625, |
| "nll_loss": 0.5, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.0, |
| "reward/margin": 1.8046875, |
| "reward/rejected": -2.8125, |
| "step": 141 |
| }, |
| { |
| "approx. KL/chosen": 34.75, |
| "approx. KL/rejected": 101.0, |
| "epoch": 0.14549180327868852, |
| "grad_norm": 67.51954650878906, |
| "learning_rate": 7.957473149787225e-06, |
| "logp/chosen": -600.0, |
| "logp/rejected": -588.0, |
| "loss": 1.26416015625, |
| "nll_loss": 0.65625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.75, |
| "reward/margin": 1.4296875, |
| "reward/rejected": -2.1875, |
| "step": 142 |
| }, |
| { |
| "approx. KL/chosen": 28.0, |
| "approx. KL/rejected": 374.0, |
| "epoch": 0.14651639344262296, |
| "grad_norm": 126.96371459960938, |
| "learning_rate": 7.955476291455656e-06, |
| "logp/chosen": -1352.0, |
| "logp/rejected": -1840.0, |
| "loss": 1.5205078125, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.0, |
| "reward/margin": 2.609375, |
| "reward/rejected": -3.609375, |
| "step": 143 |
| }, |
| { |
| "approx. KL/chosen": 37.0, |
| "approx. KL/rejected": 346.0, |
| "epoch": 0.14754098360655737, |
| "grad_norm": 124.71959686279297, |
| "learning_rate": 7.953433912517853e-06, |
| "logp/chosen": -1280.0, |
| "logp/rejected": -1640.0, |
| "loss": 1.5537109375, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.3046875, |
| "reward/margin": 1.953125, |
| "reward/rejected": -3.25, |
| "step": 144 |
| }, |
| { |
| "approx. KL/chosen": 44.0, |
| "approx. KL/rejected": 312.0, |
| "epoch": 0.14856557377049182, |
| "grad_norm": 65.69171142578125, |
| "learning_rate": 7.951346039122306e-06, |
| "logp/chosen": -968.0, |
| "logp/rejected": -1520.0, |
| "loss": 0.798583984375, |
| "nll_loss": 0.50390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.40234375, |
| "reward/margin": 3.515625, |
| "reward/rejected": -3.90625, |
| "step": 145 |
| }, |
| { |
| "approx. KL/chosen": 25.0, |
| "approx. KL/rejected": 142.0, |
| "epoch": 0.14959016393442623, |
| "grad_norm": 76.0444107055664, |
| "learning_rate": 7.949212697999965e-06, |
| "logp/chosen": -948.0, |
| "logp/rejected": -1064.0, |
| "loss": 1.154296875, |
| "nll_loss": 0.490234375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.67578125, |
| "reward/margin": 1.453125, |
| "reward/rejected": -2.125, |
| "step": 146 |
| }, |
| { |
| "approx. KL/chosen": 9.0625, |
| "approx. KL/rejected": 106.5, |
| "epoch": 0.15061475409836064, |
| "grad_norm": 74.3414077758789, |
| "learning_rate": 7.947033916463901e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -976.0, |
| "loss": 1.0966796875, |
| "nll_loss": 0.640625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.52734375, |
| "reward/margin": 1.5703125, |
| "reward/rejected": -2.09375, |
| "step": 147 |
| }, |
| { |
| "approx. KL/chosen": 42.25, |
| "approx. KL/rejected": 205.0, |
| "epoch": 0.15163934426229508, |
| "grad_norm": 84.32405090332031, |
| "learning_rate": 7.94480972240896e-06, |
| "logp/chosen": -536.0, |
| "logp/rejected": -1096.0, |
| "loss": 1.21337890625, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.75390625, |
| "reward/margin": 2.015625, |
| "reward/rejected": -2.765625, |
| "step": 148 |
| }, |
| { |
| "approx. KL/chosen": 40.0, |
| "approx. KL/rejected": 157.0, |
| "epoch": 0.1526639344262295, |
| "grad_norm": 42.48150634765625, |
| "learning_rate": 7.942540144311402e-06, |
| "logp/chosen": -1168.0, |
| "logp/rejected": -1656.0, |
| "loss": 0.77685546875, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.0, |
| "reward/margin": 2.90625, |
| "reward/rejected": -2.90625, |
| "step": 149 |
| }, |
| { |
| "approx. KL/chosen": 24.0, |
| "approx. KL/rejected": 163.0, |
| "epoch": 0.15368852459016394, |
| "grad_norm": 129.36358642578125, |
| "learning_rate": 7.94022521122853e-06, |
| "logp/chosen": -1120.0, |
| "logp/rejected": -1184.0, |
| "loss": 1.58935546875, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 0.90234375, |
| "reward/rejected": -1.703125, |
| "step": 150 |
| }, |
| { |
| "approx. KL/chosen": 29.0, |
| "approx. KL/rejected": 62.0, |
| "epoch": 0.15471311475409835, |
| "grad_norm": 85.20651245117188, |
| "learning_rate": 7.937864952798337e-06, |
| "logp/chosen": -660.0, |
| "logp/rejected": -824.0, |
| "loss": 1.25146484375, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 1.3515625, |
| "reward/rejected": -1.6484375, |
| "step": 151 |
| }, |
| { |
| "approx. KL/chosen": 10.25, |
| "approx. KL/rejected": 77.0, |
| "epoch": 0.1557377049180328, |
| "grad_norm": 95.46196746826172, |
| "learning_rate": 7.935459399239102e-06, |
| "logp/chosen": -880.0, |
| "logp/rejected": -1048.0, |
| "loss": 1.3154296875, |
| "nll_loss": 0.431640625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 0.6640625, |
| "reward/rejected": -1.3125, |
| "step": 152 |
| }, |
| { |
| "approx. KL/chosen": 44.25, |
| "approx. KL/rejected": 56.0, |
| "epoch": 0.1567622950819672, |
| "grad_norm": 82.42704772949219, |
| "learning_rate": 7.933008581349022e-06, |
| "logp/chosen": -964.0, |
| "logp/rejected": -1088.0, |
| "loss": 1.466796875, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.451171875, |
| "reward/margin": 0.75, |
| "reward/rejected": -1.203125, |
| "step": 153 |
| }, |
| { |
| "approx. KL/chosen": 25.0, |
| "approx. KL/rejected": 172.0, |
| "epoch": 0.15778688524590165, |
| "grad_norm": 44.82243728637695, |
| "learning_rate": 7.930512530505813e-06, |
| "logp/chosen": -1248.0, |
| "logp/rejected": -1520.0, |
| "loss": 0.8642578125, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 2.90625, |
| "reward/rejected": -3.0, |
| "step": 154 |
| }, |
| { |
| "approx. KL/chosen": 2.5, |
| "approx. KL/rejected": 101.0, |
| "epoch": 0.15881147540983606, |
| "grad_norm": 55.1143798828125, |
| "learning_rate": 7.927971278666303e-06, |
| "logp/chosen": -844.0, |
| "logp/rejected": -1216.0, |
| "loss": 1.02783203125, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 1.5859375, |
| "reward/rejected": -1.890625, |
| "step": 155 |
| }, |
| { |
| "approx. KL/chosen": 47.0, |
| "approx. KL/rejected": 138.0, |
| "epoch": 0.1598360655737705, |
| "grad_norm": 51.267086029052734, |
| "learning_rate": 7.925384858366026e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1600.0, |
| "loss": 0.80712890625, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.90234375, |
| "reward/margin": 3.5, |
| "reward/rejected": -2.609375, |
| "step": 156 |
| }, |
| { |
| "approx. KL/chosen": 11.0, |
| "approx. KL/rejected": 5.75, |
| "epoch": 0.16086065573770492, |
| "grad_norm": 63.739933013916016, |
| "learning_rate": 7.922753302718807e-06, |
| "logp/chosen": -740.0, |
| "logp/rejected": -720.0, |
| "loss": 1.22119140625, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": 0.08740234375, |
| "reward/margin": 0.4375, |
| "reward/rejected": -0.349609375, |
| "step": 157 |
| }, |
| { |
| "approx. KL/chosen": 17.5, |
| "approx. KL/rejected": 48.25, |
| "epoch": 0.16188524590163936, |
| "grad_norm": 92.93997192382812, |
| "learning_rate": 7.920076645416341e-06, |
| "logp/chosen": -768.0, |
| "logp/rejected": -964.0, |
| "loss": 1.26123046875, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.57421875, |
| "reward/margin": 0.77734375, |
| "reward/rejected": -1.3515625, |
| "step": 158 |
| }, |
| { |
| "approx. KL/chosen": 7.0, |
| "approx. KL/rejected": 38.0, |
| "epoch": 0.16290983606557377, |
| "grad_norm": 61.41268539428711, |
| "learning_rate": 7.917354920727748e-06, |
| "logp/chosen": -652.0, |
| "logp/rejected": -908.0, |
| "loss": 1.00244140625, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 1.0390625, |
| "reward/rejected": -1.140625, |
| "step": 159 |
| }, |
| { |
| "approx. KL/chosen": 42.0, |
| "approx. KL/rejected": 56.5, |
| "epoch": 0.16393442622950818, |
| "grad_norm": 122.8388671875, |
| "learning_rate": 7.914588163499152e-06, |
| "logp/chosen": -1352.0, |
| "logp/rejected": -1528.0, |
| "loss": 1.553955078125, |
| "nll_loss": 0.52734375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.2001953125, |
| "reward/margin": 1.0390625, |
| "reward/rejected": -0.83984375, |
| "step": 160 |
| }, |
| { |
| "approx. KL/chosen": 3.8125, |
| "approx. KL/rejected": 27.25, |
| "epoch": 0.16495901639344263, |
| "grad_norm": 55.39765167236328, |
| "learning_rate": 7.911776409153225e-06, |
| "logp/chosen": -632.0, |
| "logp/rejected": -620.0, |
| "loss": 0.904052734375, |
| "nll_loss": 0.484375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.1748046875, |
| "reward/margin": 0.9765625, |
| "reward/rejected": -1.1484375, |
| "step": 161 |
| }, |
| { |
| "approx. KL/chosen": 15.3125, |
| "approx. KL/rejected": 74.0, |
| "epoch": 0.16598360655737704, |
| "grad_norm": 92.99064636230469, |
| "learning_rate": 7.90891969368873e-06, |
| "logp/chosen": -972.0, |
| "logp/rejected": -1216.0, |
| "loss": 1.05419921875, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.42578125, |
| "reward/margin": 1.6015625, |
| "reward/rejected": -2.015625, |
| "step": 162 |
| }, |
| { |
| "approx. KL/chosen": 26.25, |
| "approx. KL/rejected": 92.5, |
| "epoch": 0.16700819672131148, |
| "grad_norm": 74.79086303710938, |
| "learning_rate": 7.906018053680074e-06, |
| "logp/chosen": -892.0, |
| "logp/rejected": -1368.0, |
| "loss": 1.0888671875, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.451171875, |
| "reward/margin": 1.6796875, |
| "reward/rejected": -2.125, |
| "step": 163 |
| }, |
| { |
| "approx. KL/chosen": 10.25, |
| "approx. KL/rejected": 89.0, |
| "epoch": 0.1680327868852459, |
| "grad_norm": 80.12300109863281, |
| "learning_rate": 7.903071526276821e-06, |
| "logp/chosen": -952.0, |
| "logp/rejected": -1120.0, |
| "loss": 1.141845703125, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.150390625, |
| "reward/margin": 1.5234375, |
| "reward/rejected": -1.375, |
| "step": 164 |
| }, |
| { |
| "approx. KL/chosen": 4.0625, |
| "approx. KL/rejected": 76.5, |
| "epoch": 0.16905737704918034, |
| "grad_norm": 89.45606231689453, |
| "learning_rate": 7.900080149203235e-06, |
| "logp/chosen": -1192.0, |
| "logp/rejected": -1080.0, |
| "loss": 0.927734375, |
| "nll_loss": 0.52734375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.1748046875, |
| "reward/margin": 1.5546875, |
| "reward/rejected": -1.375, |
| "step": 165 |
| }, |
| { |
| "approx. KL/chosen": 25.625, |
| "approx. KL/rejected": 157.0, |
| "epoch": 0.17008196721311475, |
| "grad_norm": 74.40506744384766, |
| "learning_rate": 7.897043960757785e-06, |
| "logp/chosen": -768.0, |
| "logp/rejected": -1144.0, |
| "loss": 1.103515625, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.1494140625, |
| "reward/margin": 2.5, |
| "reward/rejected": -2.359375, |
| "step": 166 |
| }, |
| { |
| "approx. KL/chosen": 30.0, |
| "approx. KL/rejected": 134.0, |
| "epoch": 0.1711065573770492, |
| "grad_norm": 91.499755859375, |
| "learning_rate": 7.893962999812656e-06, |
| "logp/chosen": -968.0, |
| "logp/rejected": -1256.0, |
| "loss": 1.161376953125, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.0, |
| "reward/margin": 1.8046875, |
| "reward/rejected": -1.796875, |
| "step": 167 |
| }, |
| { |
| "approx. KL/chosen": 17.0, |
| "approx. KL/rejected": 99.0, |
| "epoch": 0.1721311475409836, |
| "grad_norm": 58.250770568847656, |
| "learning_rate": 7.890837305813255e-06, |
| "logp/chosen": -664.0, |
| "logp/rejected": -1104.0, |
| "loss": 1.1748046875, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.1005859375, |
| "reward/margin": 1.75, |
| "reward/rejected": -1.8515625, |
| "step": 168 |
| }, |
| { |
| "approx. KL/chosen": 6.625, |
| "approx. KL/rejected": 68.0, |
| "epoch": 0.17315573770491804, |
| "grad_norm": 101.37284088134766, |
| "learning_rate": 7.887666918777706e-06, |
| "logp/chosen": -788.0, |
| "logp/rejected": -1144.0, |
| "loss": 0.9912109375, |
| "nll_loss": 0.50390625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": 0.337890625, |
| "reward/margin": 1.2890625, |
| "reward/rejected": -0.953125, |
| "step": 169 |
| }, |
| { |
| "approx. KL/chosen": 22.0, |
| "approx. KL/rejected": 41.75, |
| "epoch": 0.17418032786885246, |
| "grad_norm": 105.90069580078125, |
| "learning_rate": 7.884451879296335e-06, |
| "logp/chosen": -1192.0, |
| "logp/rejected": -1048.0, |
| "loss": 0.963623046875, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.0250244140625, |
| "reward/margin": 1.609375, |
| "reward/rejected": -1.5859375, |
| "step": 170 |
| }, |
| { |
| "approx. KL/chosen": 30.25, |
| "approx. KL/rejected": 62.0, |
| "epoch": 0.17520491803278687, |
| "grad_norm": 75.2332534790039, |
| "learning_rate": 7.88119222853115e-06, |
| "logp/chosen": -1112.0, |
| "logp/rejected": -1600.0, |
| "loss": 0.90380859375, |
| "nll_loss": 0.65625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.451171875, |
| "reward/margin": 2.046875, |
| "reward/rejected": -1.6015625, |
| "step": 171 |
| }, |
| { |
| "approx. KL/chosen": 48.25, |
| "approx. KL/rejected": 72.0, |
| "epoch": 0.1762295081967213, |
| "grad_norm": 88.57459259033203, |
| "learning_rate": 7.877888008215313e-06, |
| "logp/chosen": -964.0, |
| "logp/rejected": -1392.0, |
| "loss": 1.18896484375, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.3515625, |
| "reward/margin": 1.59375, |
| "reward/rejected": -1.2421875, |
| "step": 172 |
| }, |
| { |
| "approx. KL/chosen": 42.25, |
| "approx. KL/rejected": 105.5, |
| "epoch": 0.17725409836065573, |
| "grad_norm": 35.82814025878906, |
| "learning_rate": 7.874539260652615e-06, |
| "logp/chosen": -608.0, |
| "logp/rejected": -864.0, |
| "loss": 0.7548828125, |
| "nll_loss": 0.50390625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.84375, |
| "reward/margin": 2.953125, |
| "reward/rejected": -2.109375, |
| "step": 173 |
| }, |
| { |
| "approx. KL/chosen": 32.0, |
| "approx. KL/rejected": 27.25, |
| "epoch": 0.17827868852459017, |
| "grad_norm": 128.7445526123047, |
| "learning_rate": 7.871146028716923e-06, |
| "logp/chosen": -1216.0, |
| "logp/rejected": -996.0, |
| "loss": 1.3427734375, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 0.55078125, |
| "reward/rejected": -1.1484375, |
| "step": 174 |
| }, |
| { |
| "approx. KL/chosen": 73.0, |
| "approx. KL/rejected": 53.5, |
| "epoch": 0.17930327868852458, |
| "grad_norm": 143.7692108154297, |
| "learning_rate": 7.867708355851634e-06, |
| "logp/chosen": -980.0, |
| "logp/rejected": -1280.0, |
| "loss": 2.65673828125, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -0.75390625, |
| "reward/margin": -0.44921875, |
| "reward/rejected": -0.30078125, |
| "step": 175 |
| }, |
| { |
| "approx. KL/chosen": 17.75, |
| "approx. KL/rejected": 64.0, |
| "epoch": 0.18032786885245902, |
| "grad_norm": 78.90389251708984, |
| "learning_rate": 7.864226286069123e-06, |
| "logp/chosen": -892.0, |
| "logp/rejected": -1056.0, |
| "loss": 1.11474609375, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.625, |
| "reward/margin": 1.171875, |
| "reward/rejected": -1.8046875, |
| "step": 176 |
| }, |
| { |
| "approx. KL/chosen": 51.25, |
| "approx. KL/rejected": 120.5, |
| "epoch": 0.18135245901639344, |
| "grad_norm": 117.0498046875, |
| "learning_rate": 7.860699863950177e-06, |
| "logp/chosen": -944.0, |
| "logp/rejected": -852.0, |
| "loss": 2.0087890625, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.75390625, |
| "reward/margin": 0.55078125, |
| "reward/rejected": -1.296875, |
| "step": 177 |
| }, |
| { |
| "approx. KL/chosen": 18.25, |
| "approx. KL/rejected": 125.0, |
| "epoch": 0.18237704918032788, |
| "grad_norm": 81.22514343261719, |
| "learning_rate": 7.857129134643423e-06, |
| "logp/chosen": -956.0, |
| "logp/rejected": -1328.0, |
| "loss": 1.15478515625, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 1.453125, |
| "reward/rejected": -2.296875, |
| "step": 178 |
| }, |
| { |
| "approx. KL/chosen": 35.25, |
| "approx. KL/rejected": 153.0, |
| "epoch": 0.1834016393442623, |
| "grad_norm": 92.2053451538086, |
| "learning_rate": 7.853514143864748e-06, |
| "logp/chosen": -968.0, |
| "logp/rejected": -920.0, |
| "loss": 1.040771484375, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.94921875, |
| "reward/margin": 1.703125, |
| "reward/rejected": -2.65625, |
| "step": 179 |
| }, |
| { |
| "approx. KL/chosen": 82.0, |
| "approx. KL/rejected": 157.0, |
| "epoch": 0.18442622950819673, |
| "grad_norm": 170.3605194091797, |
| "learning_rate": 7.849854937896723e-06, |
| "logp/chosen": -1440.0, |
| "logp/rejected": -1120.0, |
| "loss": 2.28076171875, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.203125, |
| "reward/margin": 0.68359375, |
| "reward/rejected": -1.8828125, |
| "step": 180 |
| }, |
| { |
| "approx. KL/chosen": 21.25, |
| "approx. KL/rejected": 106.5, |
| "epoch": 0.18545081967213115, |
| "grad_norm": 97.09972381591797, |
| "learning_rate": 7.846151563588e-06, |
| "logp/chosen": -948.0, |
| "logp/rejected": -1288.0, |
| "loss": 1.349609375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.25, |
| "reward/margin": 1.578125, |
| "reward/rejected": -1.828125, |
| "step": 181 |
| }, |
| { |
| "approx. KL/chosen": 34.0, |
| "approx. KL/rejected": 27.25, |
| "epoch": 0.1864754098360656, |
| "grad_norm": 103.85174560546875, |
| "learning_rate": 7.842404068352713e-06, |
| "logp/chosen": -932.0, |
| "logp/rejected": -1400.0, |
| "loss": 1.05419921875, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.2041015625, |
| "reward/margin": 1.2578125, |
| "reward/rejected": -1.0546875, |
| "step": 182 |
| }, |
| { |
| "approx. KL/chosen": 49.5, |
| "approx. KL/rejected": 89.0, |
| "epoch": 0.1875, |
| "grad_norm": 101.20012664794922, |
| "learning_rate": 7.838612500169885e-06, |
| "logp/chosen": -896.0, |
| "logp/rejected": -900.0, |
| "loss": 1.67724609375, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 1.0546875, |
| "reward/rejected": -1.65625, |
| "step": 183 |
| }, |
| { |
| "approx. KL/chosen": 12.5625, |
| "approx. KL/rejected": 40.0, |
| "epoch": 0.1885245901639344, |
| "grad_norm": 68.28558349609375, |
| "learning_rate": 7.834776907582792e-06, |
| "logp/chosen": -676.0, |
| "logp/rejected": -712.0, |
| "loss": 1.09619140625, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.275390625, |
| "reward/margin": 1.1171875, |
| "reward/rejected": -1.390625, |
| "step": 184 |
| }, |
| { |
| "approx. KL/chosen": 26.0, |
| "approx. KL/rejected": 126.0, |
| "epoch": 0.18954918032786885, |
| "grad_norm": 88.36156463623047, |
| "learning_rate": 7.830897339698359e-06, |
| "logp/chosen": -1004.0, |
| "logp/rejected": -1504.0, |
| "loss": 1.2529296875, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.42578125, |
| "reward/margin": 1.828125, |
| "reward/rejected": -2.25, |
| "step": 185 |
| }, |
| { |
| "approx. KL/chosen": 26.25, |
| "approx. KL/rejected": 27.75, |
| "epoch": 0.19057377049180327, |
| "grad_norm": 80.64176177978516, |
| "learning_rate": 7.82697384618652e-06, |
| "logp/chosen": -680.0, |
| "logp/rejected": -920.0, |
| "loss": 1.258056640625, |
| "nll_loss": 0.46875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.26171875, |
| "reward/margin": 0.7265625, |
| "reward/rejected": -0.98828125, |
| "step": 186 |
| }, |
| { |
| "approx. KL/chosen": 8.8125, |
| "approx. KL/rejected": 28.25, |
| "epoch": 0.1915983606557377, |
| "grad_norm": 111.44073486328125, |
| "learning_rate": 7.82300647727959e-06, |
| "logp/chosen": -740.0, |
| "logp/rejected": -876.0, |
| "loss": 1.59423828125, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": 0.025390625, |
| "reward/margin": 0.275390625, |
| "reward/rejected": -0.251953125, |
| "step": 187 |
| }, |
| { |
| "approx. KL/chosen": 34.0, |
| "approx. KL/rejected": 13.3125, |
| "epoch": 0.19262295081967212, |
| "grad_norm": 80.01433563232422, |
| "learning_rate": 7.81899528377162e-06, |
| "logp/chosen": -764.0, |
| "logp/rejected": -876.0, |
| "loss": 1.37890625, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.07470703125, |
| "reward/margin": 0.349609375, |
| "reward/rejected": -0.42578125, |
| "step": 188 |
| }, |
| { |
| "approx. KL/chosen": 10.0, |
| "approx. KL/rejected": 250.0, |
| "epoch": 0.19364754098360656, |
| "grad_norm": 16.010801315307617, |
| "learning_rate": 7.814940317017744e-06, |
| "logp/chosen": -780.0, |
| "logp/rejected": -1264.0, |
| "loss": 0.629638671875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.0, |
| "reward/margin": 4.0625, |
| "reward/rejected": -4.0625, |
| "step": 189 |
| }, |
| { |
| "approx. KL/chosen": 48.25, |
| "approx. KL/rejected": 60.5, |
| "epoch": 0.19467213114754098, |
| "grad_norm": 103.32173156738281, |
| "learning_rate": 7.810841628933519e-06, |
| "logp/chosen": -780.0, |
| "logp/rejected": -1032.0, |
| "loss": 1.6298828125, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.1796875, |
| "reward/margin": 0.224609375, |
| "reward/rejected": -1.40625, |
| "step": 190 |
| }, |
| { |
| "approx. KL/chosen": 22.5, |
| "approx. KL/rejected": 99.5, |
| "epoch": 0.19569672131147542, |
| "grad_norm": 60.65747833251953, |
| "learning_rate": 7.806699271994272e-06, |
| "logp/chosen": -1216.0, |
| "logp/rejected": -1376.0, |
| "loss": 0.9912109375, |
| "nll_loss": 0.6796875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.2001953125, |
| "reward/margin": 2.28125, |
| "reward/rejected": -2.078125, |
| "step": 191 |
| }, |
| { |
| "approx. KL/chosen": 67.0, |
| "approx. KL/rejected": 37.0, |
| "epoch": 0.19672131147540983, |
| "grad_norm": 108.11658477783203, |
| "learning_rate": 7.802513299234413e-06, |
| "logp/chosen": -680.0, |
| "logp/rejected": -700.0, |
| "loss": 1.7607421875, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.078125, |
| "reward/margin": 0.0262451171875, |
| "reward/rejected": -1.1015625, |
| "step": 192 |
| }, |
| { |
| "approx. KL/chosen": 38.25, |
| "approx. KL/rejected": 258.0, |
| "epoch": 0.19774590163934427, |
| "grad_norm": 86.03533935546875, |
| "learning_rate": 7.79828376424677e-06, |
| "logp/chosen": -716.0, |
| "logp/rejected": -860.0, |
| "loss": 1.42822265625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.7265625, |
| "reward/margin": 2.3125, |
| "reward/rejected": -3.03125, |
| "step": 193 |
| }, |
| { |
| "approx. KL/chosen": 25.25, |
| "approx. KL/rejected": 120.0, |
| "epoch": 0.1987704918032787, |
| "grad_norm": 71.83589935302734, |
| "learning_rate": 7.79401072118189e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1440.0, |
| "loss": 0.98388671875, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.050048828125, |
| "reward/margin": 2.203125, |
| "reward/rejected": -2.15625, |
| "step": 194 |
| }, |
| { |
| "approx. KL/chosen": 38.0, |
| "approx. KL/rejected": 164.0, |
| "epoch": 0.19979508196721313, |
| "grad_norm": 126.16958618164062, |
| "learning_rate": 7.789694224747358e-06, |
| "logp/chosen": -1208.0, |
| "logp/rejected": -1376.0, |
| "loss": 1.68359375, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": 0.00048828125, |
| "reward/margin": 1.40625, |
| "reward/rejected": -1.40625, |
| "step": 195 |
| }, |
| { |
| "approx. KL/chosen": 21.0, |
| "approx. KL/rejected": 206.0, |
| "epoch": 0.20081967213114754, |
| "grad_norm": 98.14974975585938, |
| "learning_rate": 7.785334330207085e-06, |
| "logp/chosen": -1168.0, |
| "logp/rejected": -1472.0, |
| "loss": 1.3984375, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.474609375, |
| "reward/margin": 2.3125, |
| "reward/rejected": -2.78125, |
| "step": 196 |
| }, |
| { |
| "epoch": 0.20081967213114754, |
| "eval_approx. KL/chosen": 31.455, |
| "eval_approx. KL/rejected": 187.56, |
| "eval_logp/chosen": -967.04, |
| "eval_logp/rejected": -1289.6, |
| "eval_loss": 1.1985156536102295, |
| "eval_nll_loss": 0.555625, |
| "eval_reward/accuracy": 0.655, |
| "eval_reward/chosen": -0.43435546875, |
| "eval_reward/margin": 2.12484375, |
| "eval_reward/rejected": -2.55859375, |
| "eval_runtime": 46.7773, |
| "eval_samples_per_second": 4.233, |
| "eval_steps_per_second": 0.534, |
| "step": 196 |
| }, |
| { |
| "approx. KL/chosen": 21.125, |
| "approx. KL/rejected": 29.5, |
| "epoch": 0.20184426229508196, |
| "grad_norm": 63.02991485595703, |
| "learning_rate": 7.780931093380609e-06, |
| "logp/chosen": -768.0, |
| "logp/rejected": -632.0, |
| "loss": 1.14404296875, |
| "nll_loss": 0.466796875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.328125, |
| "reward/margin": 0.90234375, |
| "reward/rejected": -1.234375, |
| "step": 197 |
| }, |
| { |
| "approx. KL/chosen": 29.25, |
| "approx. KL/rejected": 109.0, |
| "epoch": 0.2028688524590164, |
| "grad_norm": 21.706876754760742, |
| "learning_rate": 7.776484570642378e-06, |
| "logp/chosen": -1128.0, |
| "logp/rejected": -1536.0, |
| "loss": 0.75341796875, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.92578125, |
| "reward/margin": 3.234375, |
| "reward/rejected": -2.296875, |
| "step": 198 |
| }, |
| { |
| "approx. KL/chosen": 36.25, |
| "approx. KL/rejected": 6.375, |
| "epoch": 0.2038934426229508, |
| "grad_norm": 67.90631103515625, |
| "learning_rate": 7.771994818921025e-06, |
| "logp/chosen": -600.0, |
| "logp/rejected": -572.0, |
| "loss": 1.38427734375, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.275390625, |
| "reward/margin": 0.275390625, |
| "reward/rejected": -0.55078125, |
| "step": 199 |
| }, |
| { |
| "approx. KL/chosen": 69.0, |
| "approx. KL/rejected": 69.0, |
| "epoch": 0.20491803278688525, |
| "grad_norm": 120.01561737060547, |
| "learning_rate": 7.767461895698646e-06, |
| "logp/chosen": -1224.0, |
| "logp/rejected": -1416.0, |
| "loss": 1.28271484375, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.94921875, |
| "reward/margin": 0.10009765625, |
| "reward/rejected": -1.0546875, |
| "step": 200 |
| }, |
| { |
| "approx. KL/chosen": 30.0, |
| "approx. KL/rejected": 162.0, |
| "epoch": 0.20594262295081966, |
| "grad_norm": 66.57347106933594, |
| "learning_rate": 7.762885859010055e-06, |
| "logp/chosen": -928.0, |
| "logp/rejected": -1464.0, |
| "loss": 0.9755859375, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.0, |
| "reward/margin": 3.0, |
| "reward/rejected": -3.0, |
| "step": 201 |
| }, |
| { |
| "approx. KL/chosen": 39.25, |
| "approx. KL/rejected": 88.0, |
| "epoch": 0.2069672131147541, |
| "grad_norm": 112.23126983642578, |
| "learning_rate": 7.758266767442049e-06, |
| "logp/chosen": -1280.0, |
| "logp/rejected": -1544.0, |
| "loss": 1.2919921875, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0546875, |
| "reward/margin": 1.0, |
| "reward/rejected": -2.046875, |
| "step": 202 |
| }, |
| { |
| "approx. KL/chosen": 21.25, |
| "approx. KL/rejected": 116.5, |
| "epoch": 0.20799180327868852, |
| "grad_norm": 75.82797241210938, |
| "learning_rate": 7.753604680132653e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1368.0, |
| "loss": 1.12060546875, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.349609375, |
| "reward/margin": 2.15625, |
| "reward/rejected": -1.8046875, |
| "step": 203 |
| }, |
| { |
| "approx. KL/chosen": 20.25, |
| "approx. KL/rejected": 185.0, |
| "epoch": 0.20901639344262296, |
| "grad_norm": 82.33856964111328, |
| "learning_rate": 7.748899656770364e-06, |
| "logp/chosen": -820.0, |
| "logp/rejected": -1504.0, |
| "loss": 1.3095703125, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.150390625, |
| "reward/margin": 2.65625, |
| "reward/rejected": -2.5, |
| "step": 204 |
| }, |
| { |
| "approx. KL/chosen": 30.125, |
| "approx. KL/rejected": 100.0, |
| "epoch": 0.21004098360655737, |
| "grad_norm": 108.07193756103516, |
| "learning_rate": 7.744151757593392e-06, |
| "logp/chosen": -1200.0, |
| "logp/rejected": -1416.0, |
| "loss": 1.189453125, |
| "nll_loss": 0.5078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.1630859375, |
| "reward/margin": 2.015625, |
| "reward/rejected": -1.8515625, |
| "step": 205 |
| }, |
| { |
| "approx. KL/chosen": 9.625, |
| "approx. KL/rejected": 75.5, |
| "epoch": 0.21106557377049182, |
| "grad_norm": 40.6665153503418, |
| "learning_rate": 7.73936104338888e-06, |
| "logp/chosen": -840.0, |
| "logp/rejected": -1120.0, |
| "loss": 0.85498046875, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 1.7265625, |
| "reward/rejected": -1.875, |
| "step": 206 |
| }, |
| { |
| "approx. KL/chosen": 35.0, |
| "approx. KL/rejected": 122.0, |
| "epoch": 0.21209016393442623, |
| "grad_norm": 135.1760711669922, |
| "learning_rate": 7.734527575492129e-06, |
| "logp/chosen": -1192.0, |
| "logp/rejected": -1272.0, |
| "loss": 2.0107421875, |
| "nll_loss": 0.494140625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 0.30078125, |
| "reward/rejected": -1.203125, |
| "step": 207 |
| }, |
| { |
| "approx. KL/chosen": 24.0, |
| "approx. KL/rejected": 142.0, |
| "epoch": 0.21311475409836064, |
| "grad_norm": 153.7419891357422, |
| "learning_rate": 7.729651415785818e-06, |
| "logp/chosen": -1144.0, |
| "logp/rejected": -1608.0, |
| "loss": 1.35400390625, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.42578125, |
| "reward/margin": 1.8984375, |
| "reward/rejected": -2.328125, |
| "step": 208 |
| }, |
| { |
| "approx. KL/chosen": 5.25, |
| "approx. KL/rejected": 246.0, |
| "epoch": 0.21413934426229508, |
| "grad_norm": 92.12176513671875, |
| "learning_rate": 7.724732626699203e-06, |
| "logp/chosen": -768.0, |
| "logp/rejected": -1328.0, |
| "loss": 1.26416015625, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.349609375, |
| "reward/margin": 2.03125, |
| "reward/rejected": -2.375, |
| "step": 209 |
| }, |
| { |
| "approx. KL/chosen": 13.0625, |
| "approx. KL/rejected": 57.75, |
| "epoch": 0.2151639344262295, |
| "grad_norm": 34.90919876098633, |
| "learning_rate": 7.719771271207323e-06, |
| "logp/chosen": -644.0, |
| "logp/rejected": -988.0, |
| "loss": 0.76416015625, |
| "nll_loss": 0.470703125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.326171875, |
| "reward/margin": 1.6640625, |
| "reward/rejected": -1.34375, |
| "step": 210 |
| }, |
| { |
| "approx. KL/chosen": 18.875, |
| "approx. KL/rejected": 140.0, |
| "epoch": 0.21618852459016394, |
| "grad_norm": 34.258033752441406, |
| "learning_rate": 7.714767412830195e-06, |
| "logp/chosen": -792.0, |
| "logp/rejected": -1128.0, |
| "loss": 0.94970703125, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.2001953125, |
| "reward/margin": 2.546875, |
| "reward/rejected": -2.359375, |
| "step": 211 |
| }, |
| { |
| "approx. KL/chosen": 9.25, |
| "approx. KL/rejected": 149.0, |
| "epoch": 0.21721311475409835, |
| "grad_norm": 96.2791976928711, |
| "learning_rate": 7.709721115631997e-06, |
| "logp/chosen": -1000.0, |
| "logp/rejected": -1336.0, |
| "loss": 0.961669921875, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.25, |
| "reward/margin": 2.5625, |
| "reward/rejected": -2.296875, |
| "step": 212 |
| }, |
| { |
| "approx. KL/chosen": 17.5, |
| "approx. KL/rejected": 26.25, |
| "epoch": 0.2182377049180328, |
| "grad_norm": 62.50215148925781, |
| "learning_rate": 7.70463244422025e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -1192.0, |
| "loss": 1.0693359375, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 0.75, |
| "reward/rejected": -0.8515625, |
| "step": 213 |
| }, |
| { |
| "approx. KL/chosen": 30.125, |
| "approx. KL/rejected": 310.0, |
| "epoch": 0.2192622950819672, |
| "grad_norm": 86.74279022216797, |
| "learning_rate": 7.69950146374499e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -1328.0, |
| "loss": 1.63330078125, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.83984375, |
| "reward/margin": 1.6875, |
| "reward/rejected": -2.515625, |
| "step": 214 |
| }, |
| { |
| "approx. KL/chosen": 6.0, |
| "approx. KL/rejected": 196.0, |
| "epoch": 0.22028688524590165, |
| "grad_norm": 65.05420684814453, |
| "learning_rate": 7.694328239897932e-06, |
| "logp/chosen": -604.0, |
| "logp/rejected": -744.0, |
| "loss": 1.13037109375, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.2001953125, |
| "reward/margin": 2.203125, |
| "reward/rejected": -2.0, |
| "step": 215 |
| }, |
| { |
| "approx. KL/chosen": 8.25, |
| "approx. KL/rejected": 185.0, |
| "epoch": 0.22131147540983606, |
| "grad_norm": 60.94325637817383, |
| "learning_rate": 7.689112838911634e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1632.0, |
| "loss": 0.88623046875, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.0498046875, |
| "reward/margin": 2.578125, |
| "reward/rejected": -2.53125, |
| "step": 216 |
| }, |
| { |
| "approx. KL/chosen": 28.25, |
| "approx. KL/rejected": 137.0, |
| "epoch": 0.2223360655737705, |
| "grad_norm": 50.33382034301758, |
| "learning_rate": 7.683855327558647e-06, |
| "logp/chosen": -824.0, |
| "logp/rejected": -1032.0, |
| "loss": 0.961181640625, |
| "nll_loss": 0.373046875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.474609375, |
| "reward/margin": 1.203125, |
| "reward/rejected": -1.6796875, |
| "step": 217 |
| }, |
| { |
| "approx. KL/chosen": 22.75, |
| "approx. KL/rejected": 129.0, |
| "epoch": 0.22336065573770492, |
| "grad_norm": 59.854408264160156, |
| "learning_rate": 7.678555773150654e-06, |
| "logp/chosen": -1000.0, |
| "logp/rejected": -1520.0, |
| "loss": 1.0693359375, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.625, |
| "reward/margin": 1.7265625, |
| "reward/rejected": -2.359375, |
| "step": 218 |
| }, |
| { |
| "approx. KL/chosen": 25.0, |
| "approx. KL/rejected": 36.0, |
| "epoch": 0.22438524590163936, |
| "grad_norm": 77.44901275634766, |
| "learning_rate": 7.673214243537617e-06, |
| "logp/chosen": -748.0, |
| "logp/rejected": -900.0, |
| "loss": 1.18603515625, |
| "nll_loss": 0.46875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.52734375, |
| "reward/margin": 0.439453125, |
| "reward/rejected": -0.96484375, |
| "step": 219 |
| }, |
| { |
| "approx. KL/chosen": 43.25, |
| "approx. KL/rejected": 174.0, |
| "epoch": 0.22540983606557377, |
| "grad_norm": 67.0965805053711, |
| "learning_rate": 7.667830807106904e-06, |
| "logp/chosen": -820.0, |
| "logp/rejected": -992.0, |
| "loss": 1.08984375, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.92578125, |
| "reward/margin": 1.90625, |
| "reward/rejected": -2.828125, |
| "step": 220 |
| }, |
| { |
| "approx. KL/chosen": 13.4375, |
| "approx. KL/rejected": 82.0, |
| "epoch": 0.22643442622950818, |
| "grad_norm": 67.90257263183594, |
| "learning_rate": 7.662405532782416e-06, |
| "logp/chosen": -736.0, |
| "logp/rejected": -796.0, |
| "loss": 1.05859375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.1376953125, |
| "reward/margin": 1.390625, |
| "reward/rejected": -1.25, |
| "step": 221 |
| }, |
| { |
| "approx. KL/chosen": 57.5, |
| "approx. KL/rejected": 105.0, |
| "epoch": 0.22745901639344263, |
| "grad_norm": 102.74434661865234, |
| "learning_rate": 7.656938490023702e-06, |
| "logp/chosen": -996.0, |
| "logp/rejected": -1472.0, |
| "loss": 1.3837890625, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.6015625, |
| "reward/margin": 0.498046875, |
| "reward/rejected": -2.109375, |
| "step": 222 |
| }, |
| { |
| "approx. KL/chosen": 13.25, |
| "approx. KL/rejected": 160.0, |
| "epoch": 0.22848360655737704, |
| "grad_norm": 42.35068130493164, |
| "learning_rate": 7.651429748825066e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1200.0, |
| "loss": 0.8505859375, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.050048828125, |
| "reward/margin": 2.5625, |
| "reward/rejected": -2.609375, |
| "step": 223 |
| }, |
| { |
| "approx. KL/chosen": 89.0, |
| "approx. KL/rejected": 185.0, |
| "epoch": 0.22950819672131148, |
| "grad_norm": 139.92857360839844, |
| "learning_rate": 7.64587937971468e-06, |
| "logp/chosen": -1664.0, |
| "logp/rejected": -1728.0, |
| "loss": 1.35986328125, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 1.75, |
| "reward/rejected": -2.859375, |
| "step": 224 |
| }, |
| { |
| "approx. KL/chosen": 102.0, |
| "approx. KL/rejected": 154.0, |
| "epoch": 0.2305327868852459, |
| "grad_norm": 135.84658813476562, |
| "learning_rate": 7.640287453753676e-06, |
| "logp/chosen": -1080.0, |
| "logp/rejected": -1048.0, |
| "loss": 2.5810546875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.40625, |
| "reward/margin": -0.37890625, |
| "reward/rejected": -1.0234375, |
| "step": 225 |
| }, |
| { |
| "approx. KL/chosen": 26.25, |
| "approx. KL/rejected": 352.0, |
| "epoch": 0.23155737704918034, |
| "grad_norm": 102.92218017578125, |
| "learning_rate": 7.634654042535234e-06, |
| "logp/chosen": -1208.0, |
| "logp/rejected": -1368.0, |
| "loss": 1.675048828125, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.3515625, |
| "reward/margin": 2.359375, |
| "reward/rejected": -2.703125, |
| "step": 226 |
| }, |
| { |
| "approx. KL/chosen": 9.5, |
| "approx. KL/rejected": 110.0, |
| "epoch": 0.23258196721311475, |
| "grad_norm": 55.18415832519531, |
| "learning_rate": 7.628979218183672e-06, |
| "logp/chosen": -620.0, |
| "logp/rejected": -784.0, |
| "loss": 1.09326171875, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.099609375, |
| "reward/margin": 1.75, |
| "reward/rejected": -1.8515625, |
| "step": 227 |
| }, |
| { |
| "approx. KL/chosen": 64.0, |
| "approx. KL/rejected": 181.0, |
| "epoch": 0.2336065573770492, |
| "grad_norm": 110.50291442871094, |
| "learning_rate": 7.6232630533535135e-06, |
| "logp/chosen": -1128.0, |
| "logp/rejected": -1192.0, |
| "loss": 1.540771484375, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 1.65625, |
| "reward/rejected": -2.453125, |
| "step": 228 |
| }, |
| { |
| "approx. KL/chosen": 32.5, |
| "approx. KL/rejected": 276.0, |
| "epoch": 0.2346311475409836, |
| "grad_norm": 25.7962589263916, |
| "learning_rate": 7.617505621228566e-06, |
| "logp/chosen": -812.0, |
| "logp/rejected": -1264.0, |
| "loss": 0.84228515625, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.298828125, |
| "reward/margin": 3.75, |
| "reward/rejected": -3.453125, |
| "step": 229 |
| }, |
| { |
| "approx. KL/chosen": 39.25, |
| "approx. KL/rejected": 80.0, |
| "epoch": 0.23565573770491804, |
| "grad_norm": 124.49078369140625, |
| "learning_rate": 7.61170699552098e-06, |
| "logp/chosen": -1288.0, |
| "logp/rejected": -1112.0, |
| "loss": 1.34521484375, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0546875, |
| "reward/margin": 0.953125, |
| "reward/rejected": -2.0, |
| "step": 230 |
| }, |
| { |
| "approx. KL/chosen": 64.0, |
| "approx. KL/rejected": 93.5, |
| "epoch": 0.23668032786885246, |
| "grad_norm": 109.35415649414062, |
| "learning_rate": 7.605867250470304e-06, |
| "logp/chosen": -1136.0, |
| "logp/rejected": -1504.0, |
| "loss": 1.24609375, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 1.125, |
| "reward/rejected": -1.9765625, |
| "step": 231 |
| }, |
| { |
| "approx. KL/chosen": 73.5, |
| "approx. KL/rejected": 233.0, |
| "epoch": 0.23770491803278687, |
| "grad_norm": 133.1020050048828, |
| "learning_rate": 7.599986460842535e-06, |
| "logp/chosen": -1256.0, |
| "logp/rejected": -1472.0, |
| "loss": 1.8466796875, |
| "nll_loss": 0.66015625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 1.703125, |
| "reward/rejected": -2.5, |
| "step": 232 |
| }, |
| { |
| "approx. KL/chosen": 15.0, |
| "approx. KL/rejected": 167.0, |
| "epoch": 0.2387295081967213, |
| "grad_norm": 46.81370162963867, |
| "learning_rate": 7.594064701929164e-06, |
| "logp/chosen": -928.0, |
| "logp/rejected": -1248.0, |
| "loss": 0.7333984375, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 2.59375, |
| "reward/rejected": -2.90625, |
| "step": 233 |
| }, |
| { |
| "approx. KL/chosen": 54.25, |
| "approx. KL/rejected": 191.0, |
| "epoch": 0.23975409836065573, |
| "grad_norm": 79.97216796875, |
| "learning_rate": 7.588102049546211e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1400.0, |
| "loss": 1.20458984375, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.349609375, |
| "reward/margin": 1.953125, |
| "reward/rejected": -2.3125, |
| "step": 234 |
| }, |
| { |
| "approx. KL/chosen": 86.0, |
| "approx. KL/rejected": 214.0, |
| "epoch": 0.24077868852459017, |
| "grad_norm": 113.54476165771484, |
| "learning_rate": 7.582098580033251e-06, |
| "logp/chosen": -800.0, |
| "logp/rejected": -576.0, |
| "loss": 1.93798828125, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.625, |
| "reward/margin": 1.2890625, |
| "reward/rejected": -2.921875, |
| "step": 235 |
| }, |
| { |
| "approx. KL/chosen": 18.125, |
| "approx. KL/rejected": 134.0, |
| "epoch": 0.24180327868852458, |
| "grad_norm": 73.11628723144531, |
| "learning_rate": 7.57605437025244e-06, |
| "logp/chosen": -956.0, |
| "logp/rejected": -1704.0, |
| "loss": 1.08642578125, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 1.5234375, |
| "reward/rejected": -2.375, |
| "step": 236 |
| }, |
| { |
| "approx. KL/chosen": 134.0, |
| "approx. KL/rejected": 77.0, |
| "epoch": 0.24282786885245902, |
| "grad_norm": 152.51390075683594, |
| "learning_rate": 7.569969497587528e-06, |
| "logp/chosen": -1216.0, |
| "logp/rejected": -1432.0, |
| "loss": 1.81982421875, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.203125, |
| "reward/margin": 0.75, |
| "reward/rejected": -1.953125, |
| "step": 237 |
| }, |
| { |
| "approx. KL/chosen": 6.3125, |
| "approx. KL/rejected": 400.0, |
| "epoch": 0.24385245901639344, |
| "grad_norm": 31.43732452392578, |
| "learning_rate": 7.563844039942872e-06, |
| "logp/chosen": -552.0, |
| "logp/rejected": -932.0, |
| "loss": 0.951416015625, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.0751953125, |
| "reward/margin": 3.625, |
| "reward/rejected": -3.546875, |
| "step": 238 |
| }, |
| { |
| "approx. KL/chosen": 7.0, |
| "approx. KL/rejected": 245.0, |
| "epoch": 0.24487704918032788, |
| "grad_norm": 64.03913116455078, |
| "learning_rate": 7.557678075742436e-06, |
| "logp/chosen": -1328.0, |
| "logp/rejected": -1704.0, |
| "loss": 1.03662109375, |
| "nll_loss": 0.7265625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.5, |
| "reward/margin": 3.0, |
| "reward/rejected": -2.5, |
| "step": 239 |
| }, |
| { |
| "approx. KL/chosen": 25.0, |
| "approx. KL/rejected": 228.0, |
| "epoch": 0.2459016393442623, |
| "grad_norm": 30.874853134155273, |
| "learning_rate": 7.551471683928789e-06, |
| "logp/chosen": -1080.0, |
| "logp/rejected": -1816.0, |
| "loss": 0.688720703125, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.703125, |
| "reward/margin": 3.109375, |
| "reward/rejected": -3.8125, |
| "step": 240 |
| }, |
| { |
| "approx. KL/chosen": 34.0, |
| "approx. KL/rejected": 255.0, |
| "epoch": 0.24692622950819673, |
| "grad_norm": 49.50712203979492, |
| "learning_rate": 7.545224943962089e-06, |
| "logp/chosen": -744.0, |
| "logp/rejected": -1216.0, |
| "loss": 0.79736328125, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.25, |
| "reward/margin": 3.484375, |
| "reward/rejected": -3.234375, |
| "step": 241 |
| }, |
| { |
| "approx. KL/chosen": 73.0, |
| "approx. KL/rejected": 108.5, |
| "epoch": 0.24795081967213115, |
| "grad_norm": 159.10203552246094, |
| "learning_rate": 7.538937935819077e-06, |
| "logp/chosen": -984.0, |
| "logp/rejected": -1096.0, |
| "loss": 1.845703125, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.92578125, |
| "reward/margin": 0.55078125, |
| "reward/rejected": -1.4765625, |
| "step": 242 |
| }, |
| { |
| "approx. KL/chosen": 15.25, |
| "approx. KL/rejected": 406.0, |
| "epoch": 0.2489754098360656, |
| "grad_norm": 79.66851806640625, |
| "learning_rate": 7.532610739992037e-06, |
| "logp/chosen": -656.0, |
| "logp/rejected": -1056.0, |
| "loss": 1.2001953125, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.049560546875, |
| "reward/margin": 2.28125, |
| "reward/rejected": -2.328125, |
| "step": 243 |
| }, |
| { |
| "approx. KL/chosen": 53.25, |
| "approx. KL/rejected": 111.5, |
| "epoch": 0.25, |
| "grad_norm": 108.85737609863281, |
| "learning_rate": 7.52624343748778e-06, |
| "logp/chosen": -796.0, |
| "logp/rejected": -1004.0, |
| "loss": 1.6884765625, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.3515625, |
| "reward/margin": 0.703125, |
| "reward/rejected": -2.0625, |
| "step": 244 |
| }, |
| { |
| "approx. KL/chosen": 113.0, |
| "approx. KL/rejected": 166.0, |
| "epoch": 0.2510245901639344, |
| "grad_norm": 92.07798767089844, |
| "learning_rate": 7.5198361098266e-06, |
| "logp/chosen": -680.0, |
| "logp/rejected": -856.0, |
| "loss": 1.529296875, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.546875, |
| "reward/margin": 1.265625, |
| "reward/rejected": -2.8125, |
| "step": 245 |
| }, |
| { |
| "approx. KL/chosen": 25.25, |
| "approx. KL/rejected": 68.5, |
| "epoch": 0.2520491803278688, |
| "grad_norm": 81.78306579589844, |
| "learning_rate": 7.513388839041229e-06, |
| "logp/chosen": -984.0, |
| "logp/rejected": -932.0, |
| "loss": 1.5712890625, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.0546875, |
| "reward/margin": 0.451171875, |
| "reward/rejected": -1.5, |
| "step": 246 |
| }, |
| { |
| "approx. KL/chosen": 33.25, |
| "approx. KL/rejected": 50.5, |
| "epoch": 0.2530737704918033, |
| "grad_norm": 111.76936340332031, |
| "learning_rate": 7.506901707675791e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -916.0, |
| "loss": 1.63525390625, |
| "nll_loss": 0.474609375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.91796875, |
| "reward/margin": 0.005126953125, |
| "reward/rejected": -0.92578125, |
| "step": 247 |
| }, |
| { |
| "approx. KL/chosen": 57.25, |
| "approx. KL/rejected": 69.0, |
| "epoch": 0.2540983606557377, |
| "grad_norm": 107.21363067626953, |
| "learning_rate": 7.5003747987847435e-06, |
| "logp/chosen": -928.0, |
| "logp/rejected": -1096.0, |
| "loss": 1.37060546875, |
| "nll_loss": 0.5, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.0625, |
| "reward/margin": 0.8359375, |
| "reward/rejected": -1.8984375, |
| "step": 248 |
| }, |
| { |
| "approx. KL/chosen": 45.0, |
| "approx. KL/rejected": 490.0, |
| "epoch": 0.2551229508196721, |
| "grad_norm": 137.05825805664062, |
| "learning_rate": 7.493808195931814e-06, |
| "logp/chosen": -1312.0, |
| "logp/rejected": -1608.0, |
| "loss": 1.31494140625, |
| "nll_loss": 0.49609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 3.375, |
| "reward/rejected": -4.46875, |
| "step": 249 |
| }, |
| { |
| "approx. KL/chosen": 22.25, |
| "approx. KL/rejected": 206.0, |
| "epoch": 0.25614754098360654, |
| "grad_norm": 109.71910095214844, |
| "learning_rate": 7.487201983188927e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1176.0, |
| "loss": 1.3115234375, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.75, |
| "reward/margin": 1.90625, |
| "reward/rejected": -2.65625, |
| "step": 250 |
| }, |
| { |
| "approx. KL/chosen": 31.125, |
| "approx. KL/rejected": 294.0, |
| "epoch": 0.257172131147541, |
| "grad_norm": 62.424617767333984, |
| "learning_rate": 7.480556245135137e-06, |
| "logp/chosen": -720.0, |
| "logp/rejected": -1336.0, |
| "loss": 1.101318359375, |
| "nll_loss": 0.4765625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.3203125, |
| "reward/margin": 2.65625, |
| "reward/rejected": -2.984375, |
| "step": 251 |
| }, |
| { |
| "approx. KL/chosen": 10.0625, |
| "approx. KL/rejected": 109.0, |
| "epoch": 0.2581967213114754, |
| "grad_norm": 58.07337188720703, |
| "learning_rate": 7.473871066855531e-06, |
| "logp/chosen": -888.0, |
| "logp/rejected": -1184.0, |
| "loss": 0.95458984375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.57421875, |
| "reward/margin": 2.125, |
| "reward/rejected": -1.5546875, |
| "step": 252 |
| }, |
| { |
| "approx. KL/chosen": 8.5625, |
| "approx. KL/rejected": 63.5, |
| "epoch": 0.25922131147540983, |
| "grad_norm": 31.149999618530273, |
| "learning_rate": 7.467146533940154e-06, |
| "logp/chosen": -588.0, |
| "logp/rejected": -664.0, |
| "loss": 0.8447265625, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.0247802734375, |
| "reward/margin": 1.8984375, |
| "reward/rejected": -1.875, |
| "step": 253 |
| }, |
| { |
| "approx. KL/chosen": 41.25, |
| "approx. KL/rejected": 122.0, |
| "epoch": 0.26024590163934425, |
| "grad_norm": 77.8751220703125, |
| "learning_rate": 7.460382732482904e-06, |
| "logp/chosen": -872.0, |
| "logp/rejected": -1352.0, |
| "loss": 0.98193359375, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.349609375, |
| "reward/margin": 2.046875, |
| "reward/rejected": -2.40625, |
| "step": 254 |
| }, |
| { |
| "approx. KL/chosen": 17.25, |
| "approx. KL/rejected": 204.0, |
| "epoch": 0.2612704918032787, |
| "grad_norm": 87.2112808227539, |
| "learning_rate": 7.453579749080434e-06, |
| "logp/chosen": -960.0, |
| "logp/rejected": -1400.0, |
| "loss": 1.17431640625, |
| "nll_loss": 0.6640625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.25, |
| "reward/margin": 2.5, |
| "reward/rejected": -2.25, |
| "step": 255 |
| }, |
| { |
| "approx. KL/chosen": 45.5, |
| "approx. KL/rejected": 71.0, |
| "epoch": 0.26229508196721313, |
| "grad_norm": 31.85320472717285, |
| "learning_rate": 7.446737670831038e-06, |
| "logp/chosen": -668.0, |
| "logp/rejected": -952.0, |
| "loss": 0.85986328125, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.1826171875, |
| "reward/margin": 1.84375, |
| "reward/rejected": -1.65625, |
| "step": 256 |
| }, |
| { |
| "approx. KL/chosen": 11.0625, |
| "approx. KL/rejected": 190.0, |
| "epoch": 0.26331967213114754, |
| "grad_norm": 86.28816223144531, |
| "learning_rate": 7.4398565853335456e-06, |
| "logp/chosen": -848.0, |
| "logp/rejected": -1192.0, |
| "loss": 1.361328125, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.52734375, |
| "reward/margin": 1.375, |
| "reward/rejected": -1.90625, |
| "step": 257 |
| }, |
| { |
| "approx. KL/chosen": 102.0, |
| "approx. KL/rejected": 71.5, |
| "epoch": 0.26434426229508196, |
| "grad_norm": 114.98932647705078, |
| "learning_rate": 7.43293658068619e-06, |
| "logp/chosen": -940.0, |
| "logp/rejected": -860.0, |
| "loss": 2.00341796875, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.8046875, |
| "reward/margin": -0.2255859375, |
| "reward/rejected": -1.578125, |
| "step": 258 |
| }, |
| { |
| "approx. KL/chosen": 23.25, |
| "approx. KL/rejected": 221.0, |
| "epoch": 0.26536885245901637, |
| "grad_norm": 74.52315521240234, |
| "learning_rate": 7.4259777454854835e-06, |
| "logp/chosen": -1008.0, |
| "logp/rejected": -1112.0, |
| "loss": 1.17626953125, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.050537109375, |
| "reward/margin": 2.21875, |
| "reward/rejected": -2.171875, |
| "step": 259 |
| }, |
| { |
| "approx. KL/chosen": 154.0, |
| "approx. KL/rejected": 214.0, |
| "epoch": 0.26639344262295084, |
| "grad_norm": 119.43769073486328, |
| "learning_rate": 7.4189801688250885e-06, |
| "logp/chosen": -932.0, |
| "logp/rejected": -1012.0, |
| "loss": 2.12255859375, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.6953125, |
| "reward/margin": 1.109375, |
| "reward/rejected": -2.8125, |
| "step": 260 |
| }, |
| { |
| "approx. KL/chosen": 69.0, |
| "approx. KL/rejected": 187.0, |
| "epoch": 0.26741803278688525, |
| "grad_norm": 90.503662109375, |
| "learning_rate": 7.41194394029467e-06, |
| "logp/chosen": -1072.0, |
| "logp/rejected": -1168.0, |
| "loss": 1.56298828125, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.94921875, |
| "reward/margin": 1.6640625, |
| "reward/rejected": -2.609375, |
| "step": 261 |
| }, |
| { |
| "approx. KL/chosen": 28.25, |
| "approx. KL/rejected": 204.0, |
| "epoch": 0.26844262295081966, |
| "grad_norm": 118.93916320800781, |
| "learning_rate": 7.404869149978752e-06, |
| "logp/chosen": -1128.0, |
| "logp/rejected": -1248.0, |
| "loss": 1.97216796875, |
| "nll_loss": 0.5, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.0390625, |
| "reward/margin": 0.5546875, |
| "reward/rejected": -1.59375, |
| "step": 262 |
| }, |
| { |
| "approx. KL/chosen": 18.25, |
| "approx. KL/rejected": 146.0, |
| "epoch": 0.2694672131147541, |
| "grad_norm": 30.654476165771484, |
| "learning_rate": 7.397755888455563e-06, |
| "logp/chosen": -672.0, |
| "logp/rejected": -904.0, |
| "loss": 0.72509765625, |
| "nll_loss": 0.482421875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.375, |
| "reward/margin": 2.5625, |
| "reward/rejected": -2.1875, |
| "step": 263 |
| }, |
| { |
| "approx. KL/chosen": 164.0, |
| "approx. KL/rejected": 59.75, |
| "epoch": 0.27049180327868855, |
| "grad_norm": 135.53204345703125, |
| "learning_rate": 7.390604246795876e-06, |
| "logp/chosen": -1568.0, |
| "logp/rejected": -864.0, |
| "loss": 1.989013671875, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.15625, |
| "reward/margin": 0.5859375, |
| "reward/rejected": -1.7421875, |
| "step": 264 |
| }, |
| { |
| "approx. KL/chosen": 34.0, |
| "approx. KL/rejected": 52.0, |
| "epoch": 0.27151639344262296, |
| "grad_norm": 107.32534790039062, |
| "learning_rate": 7.383414316561845e-06, |
| "logp/chosen": -1096.0, |
| "logp/rejected": -896.0, |
| "loss": 1.62451171875, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.625, |
| "reward/margin": 0.287109375, |
| "reward/rejected": -0.9140625, |
| "step": 265 |
| }, |
| { |
| "approx. KL/chosen": 37.5, |
| "approx. KL/rejected": 114.0, |
| "epoch": 0.2725409836065574, |
| "grad_norm": 69.7734146118164, |
| "learning_rate": 7.37618618980583e-06, |
| "logp/chosen": -712.0, |
| "logp/rejected": -976.0, |
| "loss": 1.2421875, |
| "nll_loss": 0.474609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.7734375, |
| "reward/margin": 1.6015625, |
| "reward/rejected": -2.375, |
| "step": 266 |
| }, |
| { |
| "approx. KL/chosen": 24.0, |
| "approx. KL/rejected": 78.0, |
| "epoch": 0.2735655737704918, |
| "grad_norm": 86.91935729980469, |
| "learning_rate": 7.368919959069218e-06, |
| "logp/chosen": -908.0, |
| "logp/rejected": -1024.0, |
| "loss": 1.235107421875, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 1.40625, |
| "reward/rejected": -1.6015625, |
| "step": 267 |
| }, |
| { |
| "approx. KL/chosen": 46.25, |
| "approx. KL/rejected": 268.0, |
| "epoch": 0.27459016393442626, |
| "grad_norm": 119.81461334228516, |
| "learning_rate": 7.361615717381241e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -1248.0, |
| "loss": 1.505859375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 2.0, |
| "reward/rejected": -2.84375, |
| "step": 268 |
| }, |
| { |
| "approx. KL/chosen": 3.328125, |
| "approx. KL/rejected": 27.5, |
| "epoch": 0.27561475409836067, |
| "grad_norm": 39.581417083740234, |
| "learning_rate": 7.354273558257784e-06, |
| "logp/chosen": -776.0, |
| "logp/rejected": -840.0, |
| "loss": 1.04296875, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.1376953125, |
| "reward/margin": 0.94921875, |
| "reward/rejected": -1.0859375, |
| "step": 269 |
| }, |
| { |
| "approx. KL/chosen": 33.0, |
| "approx. KL/rejected": 182.0, |
| "epoch": 0.2766393442622951, |
| "grad_norm": 96.95938873291016, |
| "learning_rate": 7.346893575700187e-06, |
| "logp/chosen": -892.0, |
| "logp/rejected": -1288.0, |
| "loss": 1.222412109375, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.57421875, |
| "reward/margin": 2.28125, |
| "reward/rejected": -2.859375, |
| "step": 270 |
| }, |
| { |
| "approx. KL/chosen": 22.0, |
| "approx. KL/rejected": 338.0, |
| "epoch": 0.2776639344262295, |
| "grad_norm": 26.016515731811523, |
| "learning_rate": 7.3394758641940395e-06, |
| "logp/chosen": -744.0, |
| "logp/rejected": -1200.0, |
| "loss": 0.7119140625, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.0, |
| "reward/margin": 4.09375, |
| "reward/rejected": -4.09375, |
| "step": 271 |
| }, |
| { |
| "approx. KL/chosen": 27.5, |
| "approx. KL/rejected": 181.0, |
| "epoch": 0.2786885245901639, |
| "grad_norm": 97.75546264648438, |
| "learning_rate": 7.3320205187079785e-06, |
| "logp/chosen": -1072.0, |
| "logp/rejected": -1672.0, |
| "loss": 1.41748046875, |
| "nll_loss": 0.50390625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.375, |
| "reward/margin": 0.953125, |
| "reward/rejected": -1.328125, |
| "step": 272 |
| }, |
| { |
| "approx. KL/chosen": 60.0, |
| "approx. KL/rejected": 308.0, |
| "epoch": 0.2797131147540984, |
| "grad_norm": 89.60128021240234, |
| "learning_rate": 7.324527634692459e-06, |
| "logp/chosen": -1320.0, |
| "logp/rejected": -1600.0, |
| "loss": 1.0712890625, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.4765625, |
| "reward/margin": 3.359375, |
| "reward/rejected": -3.828125, |
| "step": 273 |
| }, |
| { |
| "approx. KL/chosen": 9.875, |
| "approx. KL/rejected": 172.0, |
| "epoch": 0.2807377049180328, |
| "grad_norm": 22.300264358520508, |
| "learning_rate": 7.3169973080785475e-06, |
| "logp/chosen": -656.0, |
| "logp/rejected": -1480.0, |
| "loss": 0.7958984375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.1943359375, |
| "reward/margin": 2.78125, |
| "reward/rejected": -2.96875, |
| "step": 274 |
| }, |
| { |
| "approx. KL/chosen": 21.25, |
| "approx. KL/rejected": 46.5, |
| "epoch": 0.2817622950819672, |
| "grad_norm": 94.88310241699219, |
| "learning_rate": 7.309429635276684e-06, |
| "logp/chosen": -1384.0, |
| "logp/rejected": -1208.0, |
| "loss": 1.11962890625, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 1.0546875, |
| "reward/rejected": -1.6015625, |
| "step": 275 |
| }, |
| { |
| "approx. KL/chosen": 35.5, |
| "approx. KL/rejected": 40.5, |
| "epoch": 0.2827868852459016, |
| "grad_norm": 108.44193267822266, |
| "learning_rate": 7.301824713175448e-06, |
| "logp/chosen": -976.0, |
| "logp/rejected": -1152.0, |
| "loss": 1.9892578125, |
| "nll_loss": 0.640625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.0, |
| "reward/margin": -0.2001953125, |
| "reward/rejected": -0.80078125, |
| "step": 276 |
| }, |
| { |
| "approx. KL/chosen": 22.0, |
| "approx. KL/rejected": 106.5, |
| "epoch": 0.2838114754098361, |
| "grad_norm": 92.51811981201172, |
| "learning_rate": 7.294182639140324e-06, |
| "logp/chosen": -1088.0, |
| "logp/rejected": -1504.0, |
| "loss": 1.26416015625, |
| "nll_loss": 0.67578125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 1.875, |
| "reward/rejected": -2.078125, |
| "step": 277 |
| }, |
| { |
| "approx. KL/chosen": 49.5, |
| "approx. KL/rejected": 304.0, |
| "epoch": 0.2848360655737705, |
| "grad_norm": 62.52113723754883, |
| "learning_rate": 7.286503511012448e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -1336.0, |
| "loss": 1.10302734375, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.203125, |
| "reward/margin": 1.5, |
| "reward/rejected": -2.703125, |
| "step": 278 |
| }, |
| { |
| "approx. KL/chosen": 60.5, |
| "approx. KL/rejected": 83.5, |
| "epoch": 0.2858606557377049, |
| "grad_norm": 88.82305145263672, |
| "learning_rate": 7.278787427107359e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -928.0, |
| "loss": 1.587890625, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0, |
| "reward/margin": 0.41015625, |
| "reward/rejected": -1.4140625, |
| "step": 279 |
| }, |
| { |
| "approx. KL/chosen": 25.0, |
| "approx. KL/rejected": 208.0, |
| "epoch": 0.28688524590163933, |
| "grad_norm": 124.61979675292969, |
| "learning_rate": 7.271034486213739e-06, |
| "logp/chosen": -896.0, |
| "logp/rejected": -1360.0, |
| "loss": 1.765625, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 2.15625, |
| "reward/rejected": -2.453125, |
| "step": 280 |
| }, |
| { |
| "approx. KL/chosen": 27.25, |
| "approx. KL/rejected": 254.0, |
| "epoch": 0.28790983606557374, |
| "grad_norm": 112.8414077758789, |
| "learning_rate": 7.263244787592148e-06, |
| "logp/chosen": -1136.0, |
| "logp/rejected": -1480.0, |
| "loss": 1.2783203125, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 2.21875, |
| "reward/rejected": -3.0625, |
| "step": 281 |
| }, |
| { |
| "approx. KL/chosen": 22.0, |
| "approx. KL/rejected": 106.0, |
| "epoch": 0.2889344262295082, |
| "grad_norm": 43.05298614501953, |
| "learning_rate": 7.255418430973754e-06, |
| "logp/chosen": -900.0, |
| "logp/rejected": -1216.0, |
| "loss": 0.837158203125, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.400390625, |
| "reward/margin": 2.953125, |
| "reward/rejected": -2.5625, |
| "step": 282 |
| }, |
| { |
| "approx. KL/chosen": 10.5, |
| "approx. KL/rejected": 26.25, |
| "epoch": 0.2899590163934426, |
| "grad_norm": 98.01559448242188, |
| "learning_rate": 7.247555516559058e-06, |
| "logp/chosen": -772.0, |
| "logp/rejected": -980.0, |
| "loss": 1.14208984375, |
| "nll_loss": 0.49609375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.5, |
| "reward/margin": 0.462890625, |
| "reward/rejected": -0.9609375, |
| "step": 283 |
| }, |
| { |
| "approx. KL/chosen": 22.5, |
| "approx. KL/rejected": 53.5, |
| "epoch": 0.29098360655737704, |
| "grad_norm": 100.65544891357422, |
| "learning_rate": 7.2396561450166035e-06, |
| "logp/chosen": -872.0, |
| "logp/rejected": -1240.0, |
| "loss": 1.11669921875, |
| "nll_loss": 0.69921875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.10009765625, |
| "reward/margin": 1.5546875, |
| "reward/rejected": -1.453125, |
| "step": 284 |
| }, |
| { |
| "approx. KL/chosen": 34.0, |
| "approx. KL/rejected": 93.0, |
| "epoch": 0.29200819672131145, |
| "grad_norm": 111.20034790039062, |
| "learning_rate": 7.231720417481699e-06, |
| "logp/chosen": -1096.0, |
| "logp/rejected": -1064.0, |
| "loss": 1.24658203125, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.0255126953125, |
| "reward/margin": 1.9296875, |
| "reward/rejected": -1.90625, |
| "step": 285 |
| }, |
| { |
| "approx. KL/chosen": 26.75, |
| "approx. KL/rejected": 242.0, |
| "epoch": 0.2930327868852459, |
| "grad_norm": 43.96208572387695, |
| "learning_rate": 7.223748435555112e-06, |
| "logp/chosen": -824.0, |
| "logp/rejected": -1192.0, |
| "loss": 0.912109375, |
| "nll_loss": 0.66015625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.07421875, |
| "reward/margin": 3.15625, |
| "reward/rejected": -3.21875, |
| "step": 286 |
| }, |
| { |
| "approx. KL/chosen": 22.75, |
| "approx. KL/rejected": 210.0, |
| "epoch": 0.29405737704918034, |
| "grad_norm": 47.573524475097656, |
| "learning_rate": 7.215740301301778e-06, |
| "logp/chosen": -836.0, |
| "logp/rejected": -1416.0, |
| "loss": 0.9619140625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.5, |
| "reward/margin": 2.515625, |
| "reward/rejected": -3.015625, |
| "step": 287 |
| }, |
| { |
| "approx. KL/chosen": 46.5, |
| "approx. KL/rejected": 98.5, |
| "epoch": 0.29508196721311475, |
| "grad_norm": 86.44419860839844, |
| "learning_rate": 7.207696117249485e-06, |
| "logp/chosen": -656.0, |
| "logp/rejected": -724.0, |
| "loss": 1.0498046875, |
| "nll_loss": 0.466796875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 1.3046875, |
| "reward/rejected": -2.203125, |
| "step": 288 |
| }, |
| { |
| "approx. KL/chosen": 47.25, |
| "approx. KL/rejected": 113.5, |
| "epoch": 0.29610655737704916, |
| "grad_norm": 254.1870574951172, |
| "learning_rate": 7.199615986387567e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1128.0, |
| "loss": 1.4111328125, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.953125, |
| "reward/margin": 1.25, |
| "reward/rejected": -2.203125, |
| "step": 289 |
| }, |
| { |
| "approx. KL/chosen": 3.8125, |
| "approx. KL/rejected": 211.0, |
| "epoch": 0.29713114754098363, |
| "grad_norm": 13.173340797424316, |
| "learning_rate": 7.191500012165584e-06, |
| "logp/chosen": -572.0, |
| "logp/rejected": -1000.0, |
| "loss": 0.75048828125, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.125, |
| "reward/margin": 3.234375, |
| "reward/rejected": -3.109375, |
| "step": 290 |
| }, |
| { |
| "approx. KL/chosen": 18.0, |
| "approx. KL/rejected": 334.0, |
| "epoch": 0.29815573770491804, |
| "grad_norm": 80.63860321044922, |
| "learning_rate": 7.1833482984919965e-06, |
| "logp/chosen": -820.0, |
| "logp/rejected": -1120.0, |
| "loss": 1.089111328125, |
| "nll_loss": 0.4765625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.388671875, |
| "reward/margin": 2.5625, |
| "reward/rejected": -2.953125, |
| "step": 291 |
| }, |
| { |
| "approx. KL/chosen": 69.5, |
| "approx. KL/rejected": 47.0, |
| "epoch": 0.29918032786885246, |
| "grad_norm": 73.36936950683594, |
| "learning_rate": 7.1751609497328335e-06, |
| "logp/chosen": -1296.0, |
| "logp/rejected": -1224.0, |
| "loss": 1.54736328125, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.087890625, |
| "reward/margin": 0.5, |
| "reward/rejected": -0.412109375, |
| "step": 292 |
| }, |
| { |
| "approx. KL/chosen": 14.25, |
| "approx. KL/rejected": 165.0, |
| "epoch": 0.30020491803278687, |
| "grad_norm": 27.525007247924805, |
| "learning_rate": 7.166938070710362e-06, |
| "logp/chosen": -760.0, |
| "logp/rejected": -1080.0, |
| "loss": 0.74072265625, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.150390625, |
| "reward/margin": 3.03125, |
| "reward/rejected": -2.890625, |
| "step": 293 |
| }, |
| { |
| "approx. KL/chosen": 50.0, |
| "approx. KL/rejected": 247.0, |
| "epoch": 0.3012295081967213, |
| "grad_norm": 82.74041748046875, |
| "learning_rate": 7.158679766701739e-06, |
| "logp/chosen": -1304.0, |
| "logp/rejected": -1752.0, |
| "loss": 0.9052734375, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.19921875, |
| "reward/margin": 2.96875, |
| "reward/rejected": -2.765625, |
| "step": 294 |
| }, |
| { |
| "approx. KL/chosen": 78.0, |
| "approx. KL/rejected": 206.0, |
| "epoch": 0.30225409836065575, |
| "grad_norm": 150.19107055664062, |
| "learning_rate": 7.150386143437667e-06, |
| "logp/chosen": -1584.0, |
| "logp/rejected": -1560.0, |
| "loss": 1.5869140625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.0546875, |
| "reward/margin": 2.15625, |
| "reward/rejected": -3.203125, |
| "step": 295 |
| }, |
| { |
| "approx. KL/chosen": 38.25, |
| "approx. KL/rejected": 328.0, |
| "epoch": 0.30327868852459017, |
| "grad_norm": 106.26634979248047, |
| "learning_rate": 7.142057307101039e-06, |
| "logp/chosen": -1024.0, |
| "logp/rejected": -1840.0, |
| "loss": 1.0927734375, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.75, |
| "reward/margin": 3.203125, |
| "reward/rejected": -3.953125, |
| "step": 296 |
| }, |
| { |
| "approx. KL/chosen": 31.0, |
| "approx. KL/rejected": 46.0, |
| "epoch": 0.3043032786885246, |
| "grad_norm": 88.35395050048828, |
| "learning_rate": 7.133693364325578e-06, |
| "logp/chosen": -980.0, |
| "logp/rejected": -1120.0, |
| "loss": 1.154541015625, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.69921875, |
| "reward/margin": 0.89453125, |
| "reward/rejected": -1.59375, |
| "step": 297 |
| }, |
| { |
| "approx. KL/chosen": 31.0, |
| "approx. KL/rejected": 58.0, |
| "epoch": 0.305327868852459, |
| "grad_norm": 89.2414321899414, |
| "learning_rate": 7.125294422194478e-06, |
| "logp/chosen": -804.0, |
| "logp/rejected": -732.0, |
| "loss": 1.46435546875, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.71484375, |
| "reward/margin": 0.8046875, |
| "reward/rejected": -1.5234375, |
| "step": 298 |
| }, |
| { |
| "approx. KL/chosen": 82.0, |
| "approx. KL/rejected": 278.0, |
| "epoch": 0.30635245901639346, |
| "grad_norm": 247.6508026123047, |
| "learning_rate": 7.116860588239023e-06, |
| "logp/chosen": -984.0, |
| "logp/rejected": -1128.0, |
| "loss": 1.765625, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.046875, |
| "reward/margin": 2.546875, |
| "reward/rejected": -3.59375, |
| "step": 299 |
| }, |
| { |
| "approx. KL/chosen": 14.4375, |
| "approx. KL/rejected": 226.0, |
| "epoch": 0.3073770491803279, |
| "grad_norm": 41.38711929321289, |
| "learning_rate": 7.108391970437215e-06, |
| "logp/chosen": -964.0, |
| "logp/rejected": -872.0, |
| "loss": 0.8642578125, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.88671875, |
| "reward/margin": 3.640625, |
| "reward/rejected": -2.75, |
| "step": 300 |
| }, |
| { |
| "approx. KL/chosen": 37.25, |
| "approx. KL/rejected": 193.0, |
| "epoch": 0.3084016393442623, |
| "grad_norm": 133.00355529785156, |
| "learning_rate": 7.0998886772123985e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1744.0, |
| "loss": 2.0234375, |
| "nll_loss": 0.640625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.349609375, |
| "reward/margin": 1.953125, |
| "reward/rejected": -2.296875, |
| "step": 301 |
| }, |
| { |
| "approx. KL/chosen": 22.0, |
| "approx. KL/rejected": 182.0, |
| "epoch": 0.3094262295081967, |
| "grad_norm": 17.467952728271484, |
| "learning_rate": 7.09135081743186e-06, |
| "logp/chosen": -592.0, |
| "logp/rejected": -680.0, |
| "loss": 0.706787109375, |
| "nll_loss": 0.50390625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.263671875, |
| "reward/margin": 3.234375, |
| "reward/rejected": -2.984375, |
| "step": 302 |
| }, |
| { |
| "approx. KL/chosen": 31.25, |
| "approx. KL/rejected": 164.0, |
| "epoch": 0.3104508196721312, |
| "grad_norm": 76.7230453491211, |
| "learning_rate": 7.082778500405441e-06, |
| "logp/chosen": -780.0, |
| "logp/rejected": -1152.0, |
| "loss": 1.50390625, |
| "nll_loss": 0.64453125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.050537109375, |
| "reward/margin": 2.390625, |
| "reward/rejected": -2.34375, |
| "step": 303 |
| }, |
| { |
| "approx. KL/chosen": 75.5, |
| "approx. KL/rejected": 216.0, |
| "epoch": 0.3114754098360656, |
| "grad_norm": 122.30782318115234, |
| "learning_rate": 7.07417183588414e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1600.0, |
| "loss": 1.4765625, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.3046875, |
| "reward/margin": 2.046875, |
| "reward/rejected": -3.34375, |
| "step": 304 |
| }, |
| { |
| "approx. KL/chosen": 10.75, |
| "approx. KL/rejected": 123.0, |
| "epoch": 0.3125, |
| "grad_norm": 72.76583099365234, |
| "learning_rate": 7.065530934058703e-06, |
| "logp/chosen": -708.0, |
| "logp/rejected": -616.0, |
| "loss": 1.43212890625, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 0.921875, |
| "reward/rejected": -1.46875, |
| "step": 305 |
| }, |
| { |
| "approx. KL/chosen": 29.75, |
| "approx. KL/rejected": 75.5, |
| "epoch": 0.3135245901639344, |
| "grad_norm": 40.34486770629883, |
| "learning_rate": 7.056855905558215e-06, |
| "logp/chosen": -772.0, |
| "logp/rejected": -712.0, |
| "loss": 0.85400390625, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.2255859375, |
| "reward/margin": 1.9765625, |
| "reward/rejected": -2.203125, |
| "step": 306 |
| }, |
| { |
| "approx. KL/chosen": 59.0, |
| "approx. KL/rejected": 552.0, |
| "epoch": 0.3145491803278688, |
| "grad_norm": 18.615427017211914, |
| "learning_rate": 7.0481468614486835e-06, |
| "logp/chosen": -796.0, |
| "logp/rejected": -1448.0, |
| "loss": 0.677734375, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 1.5, |
| "reward/margin": 5.96875, |
| "reward/rejected": -4.46875, |
| "step": 307 |
| }, |
| { |
| "approx. KL/chosen": 20.25, |
| "approx. KL/rejected": 202.0, |
| "epoch": 0.3155737704918033, |
| "grad_norm": 56.26082229614258, |
| "learning_rate": 7.039403913231616e-06, |
| "logp/chosen": -940.0, |
| "logp/rejected": -732.0, |
| "loss": 1.00830078125, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.7265625, |
| "reward/margin": 2.1875, |
| "reward/rejected": -2.90625, |
| "step": 308 |
| }, |
| { |
| "approx. KL/chosen": 252.0, |
| "approx. KL/rejected": 290.0, |
| "epoch": 0.3165983606557377, |
| "grad_norm": 158.4828338623047, |
| "learning_rate": 7.0306271728425945e-06, |
| "logp/chosen": -1336.0, |
| "logp/rejected": -1272.0, |
| "loss": 3.140625, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -3.859375, |
| "reward/margin": -1.703125, |
| "reward/rejected": -2.15625, |
| "step": 309 |
| }, |
| { |
| "approx. KL/chosen": 33.25, |
| "approx. KL/rejected": 696.0, |
| "epoch": 0.3176229508196721, |
| "grad_norm": 83.53401184082031, |
| "learning_rate": 7.021816752649838e-06, |
| "logp/chosen": -844.0, |
| "logp/rejected": -1344.0, |
| "loss": 1.100830078125, |
| "nll_loss": 0.49609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.349609375, |
| "reward/margin": 5.1875, |
| "reward/rejected": -5.53125, |
| "step": 310 |
| }, |
| { |
| "approx. KL/chosen": 61.5, |
| "approx. KL/rejected": 212.0, |
| "epoch": 0.31864754098360654, |
| "grad_norm": 108.53302764892578, |
| "learning_rate": 7.012972765452767e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -1016.0, |
| "loss": 2.033203125, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.203125, |
| "reward/margin": 0.92578125, |
| "reward/rejected": -2.125, |
| "step": 311 |
| }, |
| { |
| "approx. KL/chosen": 25.5, |
| "approx. KL/rejected": 117.0, |
| "epoch": 0.319672131147541, |
| "grad_norm": 47.210365295410156, |
| "learning_rate": 7.004095324480562e-06, |
| "logp/chosen": -1056.0, |
| "logp/rejected": -976.0, |
| "loss": 1.11962890625, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.69921875, |
| "reward/margin": 1.953125, |
| "reward/rejected": -2.65625, |
| "step": 312 |
| }, |
| { |
| "approx. KL/chosen": 42.0, |
| "approx. KL/rejected": 330.0, |
| "epoch": 0.3206967213114754, |
| "grad_norm": 38.997188568115234, |
| "learning_rate": 6.995184543390707e-06, |
| "logp/chosen": -1056.0, |
| "logp/rejected": -1336.0, |
| "loss": 0.756591796875, |
| "nll_loss": 0.6484375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.2001953125, |
| "reward/margin": 4.8125, |
| "reward/rejected": -4.625, |
| "step": 313 |
| }, |
| { |
| "approx. KL/chosen": 66.0, |
| "approx. KL/rejected": 146.0, |
| "epoch": 0.32172131147540983, |
| "grad_norm": 58.158836364746094, |
| "learning_rate": 6.986240536267541e-06, |
| "logp/chosen": -992.0, |
| "logp/rejected": -1264.0, |
| "loss": 0.89013671875, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.0, |
| "reward/margin": 1.796875, |
| "reward/rejected": -2.796875, |
| "step": 314 |
| }, |
| { |
| "approx. KL/chosen": 42.25, |
| "approx. KL/rejected": 72.5, |
| "epoch": 0.32274590163934425, |
| "grad_norm": 94.97342681884766, |
| "learning_rate": 6.977263417620791e-06, |
| "logp/chosen": -672.0, |
| "logp/rejected": -696.0, |
| "loss": 1.55322265625, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.2734375, |
| "reward/margin": -0.02587890625, |
| "reward/rejected": -1.25, |
| "step": 315 |
| }, |
| { |
| "approx. KL/chosen": 24.25, |
| "approx. KL/rejected": 173.0, |
| "epoch": 0.3237704918032787, |
| "grad_norm": 69.65333557128906, |
| "learning_rate": 6.9682533023841126e-06, |
| "logp/chosen": -776.0, |
| "logp/rejected": -916.0, |
| "loss": 0.9716796875, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.5625, |
| "reward/margin": 1.75, |
| "reward/rejected": -2.3125, |
| "step": 316 |
| }, |
| { |
| "approx. KL/chosen": 34.5, |
| "approx. KL/rejected": 128.0, |
| "epoch": 0.32479508196721313, |
| "grad_norm": 77.26544189453125, |
| "learning_rate": 6.959210305913614e-06, |
| "logp/chosen": -868.0, |
| "logp/rejected": -628.0, |
| "loss": 1.26025390625, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.38671875, |
| "reward/margin": 1.75, |
| "reward/rejected": -2.140625, |
| "step": 317 |
| }, |
| { |
| "approx. KL/chosen": 40.75, |
| "approx. KL/rejected": 226.0, |
| "epoch": 0.32581967213114754, |
| "grad_norm": 92.81001281738281, |
| "learning_rate": 6.9501345439863786e-06, |
| "logp/chosen": -832.0, |
| "logp/rejected": -1112.0, |
| "loss": 1.77197265625, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.349609375, |
| "reward/margin": 1.6796875, |
| "reward/rejected": -2.03125, |
| "step": 318 |
| }, |
| { |
| "approx. KL/chosen": 56.5, |
| "approx. KL/rejected": 490.0, |
| "epoch": 0.32684426229508196, |
| "grad_norm": 83.66230010986328, |
| "learning_rate": 6.941026132798988e-06, |
| "logp/chosen": -924.0, |
| "logp/rejected": -1208.0, |
| "loss": 1.0302734375, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 3.953125, |
| "reward/rejected": -4.84375, |
| "step": 319 |
| }, |
| { |
| "approx. KL/chosen": 73.0, |
| "approx. KL/rejected": 225.0, |
| "epoch": 0.32786885245901637, |
| "grad_norm": 143.7174072265625, |
| "learning_rate": 6.93188518896603e-06, |
| "logp/chosen": -1224.0, |
| "logp/rejected": -1096.0, |
| "loss": 2.55078125, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -1.15625, |
| "reward/margin": -0.0546875, |
| "reward/rejected": -1.1015625, |
| "step": 320 |
| }, |
| { |
| "approx. KL/chosen": 173.0, |
| "approx. KL/rejected": 177.0, |
| "epoch": 0.32889344262295084, |
| "grad_norm": 130.65586853027344, |
| "learning_rate": 6.922711829518603e-06, |
| "logp/chosen": -828.0, |
| "logp/rejected": -1104.0, |
| "loss": 2.23974609375, |
| "nll_loss": 0.671875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -2.046875, |
| "reward/margin": 0.333984375, |
| "reward/rejected": -2.390625, |
| "step": 321 |
| }, |
| { |
| "approx. KL/chosen": 14.0, |
| "approx. KL/rejected": 166.0, |
| "epoch": 0.32991803278688525, |
| "grad_norm": 96.08658599853516, |
| "learning_rate": 6.913506171902825e-06, |
| "logp/chosen": -908.0, |
| "logp/rejected": -952.0, |
| "loss": 1.24560546875, |
| "nll_loss": 0.515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 1.703125, |
| "reward/rejected": -2.296875, |
| "step": 322 |
| }, |
| { |
| "approx. KL/chosen": 38.0, |
| "approx. KL/rejected": 98.5, |
| "epoch": 0.33094262295081966, |
| "grad_norm": 123.36955261230469, |
| "learning_rate": 6.904268333978327e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1368.0, |
| "loss": 1.73583984375, |
| "nll_loss": 0.6640625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0, |
| "reward/margin": 1.1171875, |
| "reward/rejected": -2.125, |
| "step": 323 |
| }, |
| { |
| "approx. KL/chosen": 122.0, |
| "approx. KL/rejected": 244.0, |
| "epoch": 0.3319672131147541, |
| "grad_norm": 88.62934875488281, |
| "learning_rate": 6.894998434016735e-06, |
| "logp/chosen": -904.0, |
| "logp/rejected": -1088.0, |
| "loss": 2.046875, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.203125, |
| "reward/margin": 1.9765625, |
| "reward/rejected": -3.171875, |
| "step": 324 |
| }, |
| { |
| "approx. KL/chosen": 101.5, |
| "approx. KL/rejected": 113.0, |
| "epoch": 0.33299180327868855, |
| "grad_norm": 92.83050537109375, |
| "learning_rate": 6.885696590700175e-06, |
| "logp/chosen": -1224.0, |
| "logp/rejected": -1192.0, |
| "loss": 1.40185546875, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.130859375, |
| "reward/margin": 2.203125, |
| "reward/rejected": -2.328125, |
| "step": 325 |
| }, |
| { |
| "approx. KL/chosen": 90.0, |
| "approx. KL/rejected": 96.0, |
| "epoch": 0.33401639344262296, |
| "grad_norm": 93.18934631347656, |
| "learning_rate": 6.876362923119734e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1320.0, |
| "loss": 1.08984375, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0, |
| "reward/margin": 1.40625, |
| "reward/rejected": -2.40625, |
| "step": 326 |
| }, |
| { |
| "approx. KL/chosen": 33.5, |
| "approx. KL/rejected": 105.0, |
| "epoch": 0.3350409836065574, |
| "grad_norm": 76.07000732421875, |
| "learning_rate": 6.866997550773948e-06, |
| "logp/chosen": -632.0, |
| "logp/rejected": -912.0, |
| "loss": 1.072265625, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 1.2734375, |
| "reward/rejected": -2.375, |
| "step": 327 |
| }, |
| { |
| "approx. KL/chosen": 117.5, |
| "approx. KL/rejected": 72.0, |
| "epoch": 0.3360655737704918, |
| "grad_norm": 138.45449829101562, |
| "learning_rate": 6.8576005935672666e-06, |
| "logp/chosen": -1256.0, |
| "logp/rejected": -1280.0, |
| "loss": 1.76025390625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.25, |
| "reward/margin": 1.0, |
| "reward/rejected": -1.25, |
| "step": 328 |
| }, |
| { |
| "approx. KL/chosen": 6.625, |
| "approx. KL/rejected": 152.0, |
| "epoch": 0.33709016393442626, |
| "grad_norm": 52.05340576171875, |
| "learning_rate": 6.8481721718085165e-06, |
| "logp/chosen": -1200.0, |
| "logp/rejected": -1232.0, |
| "loss": 0.853271484375, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 2.4375, |
| "reward/rejected": -2.546875, |
| "step": 329 |
| }, |
| { |
| "approx. KL/chosen": 13.5, |
| "approx. KL/rejected": 139.0, |
| "epoch": 0.33811475409836067, |
| "grad_norm": 47.242061614990234, |
| "learning_rate": 6.8387124062093685e-06, |
| "logp/chosen": -548.0, |
| "logp/rejected": -752.0, |
| "loss": 1.08447265625, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 1.75, |
| "reward/rejected": -2.140625, |
| "step": 330 |
| }, |
| { |
| "approx. KL/chosen": 17.5, |
| "approx. KL/rejected": 129.0, |
| "epoch": 0.3391393442622951, |
| "grad_norm": 58.15475082397461, |
| "learning_rate": 6.829221417882783e-06, |
| "logp/chosen": -1072.0, |
| "logp/rejected": -1320.0, |
| "loss": 0.80322265625, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.0751953125, |
| "reward/margin": 2.6875, |
| "reward/rejected": -2.75, |
| "step": 331 |
| }, |
| { |
| "approx. KL/chosen": 27.75, |
| "approx. KL/rejected": 65.0, |
| "epoch": 0.3401639344262295, |
| "grad_norm": 73.17526245117188, |
| "learning_rate": 6.819699328341465e-06, |
| "logp/chosen": -616.0, |
| "logp/rejected": -868.0, |
| "loss": 1.2509765625, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.0546875, |
| "reward/margin": 1.0, |
| "reward/rejected": -2.046875, |
| "step": 332 |
| }, |
| { |
| "approx. KL/chosen": 19.0, |
| "approx. KL/rejected": 93.5, |
| "epoch": 0.3411885245901639, |
| "grad_norm": 31.51212501525879, |
| "learning_rate": 6.810146259496308e-06, |
| "logp/chosen": -672.0, |
| "logp/rejected": -664.0, |
| "loss": 0.849365234375, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 1.8984375, |
| "reward/rejected": -2.0, |
| "step": 333 |
| }, |
| { |
| "approx. KL/chosen": 24.25, |
| "approx. KL/rejected": 252.0, |
| "epoch": 0.3422131147540984, |
| "grad_norm": 55.15986633300781, |
| "learning_rate": 6.800562333654832e-06, |
| "logp/chosen": -836.0, |
| "logp/rejected": -1328.0, |
| "loss": 0.8154296875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.42578125, |
| "reward/margin": 3.265625, |
| "reward/rejected": -3.6875, |
| "step": 334 |
| }, |
| { |
| "approx. KL/chosen": 61.0, |
| "approx. KL/rejected": 148.0, |
| "epoch": 0.3432377049180328, |
| "grad_norm": 64.10730743408203, |
| "learning_rate": 6.790947673519615e-06, |
| "logp/chosen": -656.0, |
| "logp/rejected": -804.0, |
| "loss": 1.347412109375, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.7265625, |
| "reward/margin": 1.5859375, |
| "reward/rejected": -2.3125, |
| "step": 335 |
| }, |
| { |
| "approx. KL/chosen": 44.0, |
| "approx. KL/rejected": 268.0, |
| "epoch": 0.3442622950819672, |
| "grad_norm": 25.888839721679688, |
| "learning_rate": 6.781302402186727e-06, |
| "logp/chosen": -1240.0, |
| "logp/rejected": -1608.0, |
| "loss": 0.77099609375, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -1.3984375, |
| "reward/margin": 2.90625, |
| "reward/rejected": -4.3125, |
| "step": 336 |
| }, |
| { |
| "approx. KL/chosen": 73.0, |
| "approx. KL/rejected": 117.0, |
| "epoch": 0.3452868852459016, |
| "grad_norm": 88.65773010253906, |
| "learning_rate": 6.771626643144155e-06, |
| "logp/chosen": -948.0, |
| "logp/rejected": -920.0, |
| "loss": 1.73388671875, |
| "nll_loss": 0.65625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.703125, |
| "reward/margin": 0.7734375, |
| "reward/rejected": -2.484375, |
| "step": 337 |
| }, |
| { |
| "approx. KL/chosen": 8.25, |
| "approx. KL/rejected": 260.0, |
| "epoch": 0.3463114754098361, |
| "grad_norm": 123.30865478515625, |
| "learning_rate": 6.761920520270212e-06, |
| "logp/chosen": -908.0, |
| "logp/rejected": -1744.0, |
| "loss": 1.314208984375, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.349609375, |
| "reward/margin": 2.34375, |
| "reward/rejected": -2.703125, |
| "step": 338 |
| }, |
| { |
| "approx. KL/chosen": 5.25, |
| "approx. KL/rejected": 163.0, |
| "epoch": 0.3473360655737705, |
| "grad_norm": 94.02284240722656, |
| "learning_rate": 6.752184157831964e-06, |
| "logp/chosen": -920.0, |
| "logp/rejected": -1088.0, |
| "loss": 1.222412109375, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.25, |
| "reward/margin": 1.703125, |
| "reward/rejected": -1.953125, |
| "step": 339 |
| }, |
| { |
| "approx. KL/chosen": 42.0, |
| "approx. KL/rejected": 141.0, |
| "epoch": 0.3483606557377049, |
| "grad_norm": 98.73439025878906, |
| "learning_rate": 6.742417680483627e-06, |
| "logp/chosen": -1144.0, |
| "logp/rejected": -1032.0, |
| "loss": 1.14892578125, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 1.703125, |
| "reward/rejected": -2.296875, |
| "step": 340 |
| }, |
| { |
| "approx. KL/chosen": 74.5, |
| "approx. KL/rejected": 112.5, |
| "epoch": 0.34938524590163933, |
| "grad_norm": 94.97835540771484, |
| "learning_rate": 6.732621213264982e-06, |
| "logp/chosen": -1072.0, |
| "logp/rejected": -1232.0, |
| "loss": 1.916015625, |
| "nll_loss": 0.68359375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.828125, |
| "reward/margin": 0.28125, |
| "reward/rejected": -2.109375, |
| "step": 341 |
| }, |
| { |
| "approx. KL/chosen": 14.0, |
| "approx. KL/rejected": 230.0, |
| "epoch": 0.35040983606557374, |
| "grad_norm": 54.43999481201172, |
| "learning_rate": 6.722794881599767e-06, |
| "logp/chosen": -964.0, |
| "logp/rejected": -1656.0, |
| "loss": 0.87841796875, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 2.8125, |
| "reward/rejected": -3.609375, |
| "step": 342 |
| }, |
| { |
| "approx. KL/chosen": 10.3125, |
| "approx. KL/rejected": 56.0, |
| "epoch": 0.3514344262295082, |
| "grad_norm": 31.872695922851562, |
| "learning_rate": 6.712938811294072e-06, |
| "logp/chosen": -672.0, |
| "logp/rejected": -876.0, |
| "loss": 0.8082275390625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.326171875, |
| "reward/margin": 1.625, |
| "reward/rejected": -1.953125, |
| "step": 343 |
| }, |
| { |
| "approx. KL/chosen": 74.5, |
| "approx. KL/rejected": 137.0, |
| "epoch": 0.3524590163934426, |
| "grad_norm": 85.06661224365234, |
| "learning_rate": 6.7030531285347315e-06, |
| "logp/chosen": -832.0, |
| "logp/rejected": -1064.0, |
| "loss": 1.095703125, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 1.734375, |
| "reward/rejected": -2.640625, |
| "step": 344 |
| }, |
| { |
| "approx. KL/chosen": 35.0, |
| "approx. KL/rejected": 82.0, |
| "epoch": 0.35348360655737704, |
| "grad_norm": 122.48806762695312, |
| "learning_rate": 6.693137959887708e-06, |
| "logp/chosen": -876.0, |
| "logp/rejected": -1248.0, |
| "loss": 1.6630859375, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 0.50390625, |
| "reward/rejected": -1.6015625, |
| "step": 345 |
| }, |
| { |
| "approx. KL/chosen": 26.0, |
| "approx. KL/rejected": 71.5, |
| "epoch": 0.35450819672131145, |
| "grad_norm": 85.81343078613281, |
| "learning_rate": 6.683193432296464e-06, |
| "logp/chosen": -736.0, |
| "logp/rejected": -868.0, |
| "loss": 1.285888671875, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.82421875, |
| "reward/margin": 0.83984375, |
| "reward/rejected": -1.6640625, |
| "step": 346 |
| }, |
| { |
| "approx. KL/chosen": 50.5, |
| "approx. KL/rejected": 152.0, |
| "epoch": 0.3555327868852459, |
| "grad_norm": 85.1299819946289, |
| "learning_rate": 6.673219673080352e-06, |
| "logp/chosen": -704.0, |
| "logp/rejected": -1120.0, |
| "loss": 1.072265625, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 1.6640625, |
| "reward/rejected": -2.765625, |
| "step": 347 |
| }, |
| { |
| "approx. KL/chosen": 6.5625, |
| "approx. KL/rejected": 324.0, |
| "epoch": 0.35655737704918034, |
| "grad_norm": 47.68248748779297, |
| "learning_rate": 6.66321680993297e-06, |
| "logp/chosen": -876.0, |
| "logp/rejected": -1576.0, |
| "loss": 0.7587890625, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.275390625, |
| "reward/margin": 4.21875, |
| "reward/rejected": -4.5, |
| "step": 348 |
| }, |
| { |
| "approx. KL/chosen": 30.25, |
| "approx. KL/rejected": 73.0, |
| "epoch": 0.35758196721311475, |
| "grad_norm": 105.18407440185547, |
| "learning_rate": 6.653184970920534e-06, |
| "logp/chosen": -904.0, |
| "logp/rejected": -1000.0, |
| "loss": 1.552734375, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.75, |
| "reward/margin": 0.69140625, |
| "reward/rejected": -1.4375, |
| "step": 349 |
| }, |
| { |
| "approx. KL/chosen": 105.0, |
| "approx. KL/rejected": 76.0, |
| "epoch": 0.35860655737704916, |
| "grad_norm": 179.28248596191406, |
| "learning_rate": 6.643124284480236e-06, |
| "logp/chosen": -1400.0, |
| "logp/rejected": -1720.0, |
| "loss": 1.78564453125, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -2.109375, |
| "reward/margin": 0.10009765625, |
| "reward/rejected": -2.203125, |
| "step": 350 |
| }, |
| { |
| "approx. KL/chosen": 34.5, |
| "approx. KL/rejected": 229.0, |
| "epoch": 0.35963114754098363, |
| "grad_norm": 81.55355072021484, |
| "learning_rate": 6.633034879418602e-06, |
| "logp/chosen": -980.0, |
| "logp/rejected": -1240.0, |
| "loss": 1.173828125, |
| "nll_loss": 0.462890625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.0, |
| "reward/margin": 1.765625, |
| "reward/rejected": -2.765625, |
| "step": 351 |
| }, |
| { |
| "approx. KL/chosen": 25.0, |
| "approx. KL/rejected": 406.0, |
| "epoch": 0.36065573770491804, |
| "grad_norm": 79.1892318725586, |
| "learning_rate": 6.622916884909841e-06, |
| "logp/chosen": -1232.0, |
| "logp/rejected": -1360.0, |
| "loss": 1.123291015625, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 3.28125, |
| "reward/rejected": -4.1875, |
| "step": 352 |
| }, |
| { |
| "approx. KL/chosen": 101.0, |
| "approx. KL/rejected": 120.5, |
| "epoch": 0.36168032786885246, |
| "grad_norm": 123.76941680908203, |
| "learning_rate": 6.612770430494189e-06, |
| "logp/chosen": -1144.0, |
| "logp/rejected": -1272.0, |
| "loss": 1.59814453125, |
| "nll_loss": 0.5078125, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -1.9296875, |
| "reward/margin": 0.038330078125, |
| "reward/rejected": -1.96875, |
| "step": 353 |
| }, |
| { |
| "approx. KL/chosen": 21.0, |
| "approx. KL/rejected": 428.0, |
| "epoch": 0.36270491803278687, |
| "grad_norm": 68.7950668334961, |
| "learning_rate": 6.602595646076256e-06, |
| "logp/chosen": -820.0, |
| "logp/rejected": -1344.0, |
| "loss": 1.162109375, |
| "nll_loss": 0.68359375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.5, |
| "reward/margin": 3.859375, |
| "reward/rejected": -4.34375, |
| "step": 354 |
| }, |
| { |
| "approx. KL/chosen": 58.75, |
| "approx. KL/rejected": 844.0, |
| "epoch": 0.3637295081967213, |
| "grad_norm": 49.67341613769531, |
| "learning_rate": 6.592392661923356e-06, |
| "logp/chosen": -1012.0, |
| "logp/rejected": -1352.0, |
| "loss": 1.01220703125, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.0546875, |
| "reward/margin": 4.8125, |
| "reward/rejected": -5.84375, |
| "step": 355 |
| }, |
| { |
| "approx. KL/chosen": 90.0, |
| "approx. KL/rejected": 368.0, |
| "epoch": 0.36475409836065575, |
| "grad_norm": 141.9888153076172, |
| "learning_rate": 6.582161608663849e-06, |
| "logp/chosen": -1080.0, |
| "logp/rejected": -1560.0, |
| "loss": 1.676513671875, |
| "nll_loss": 0.5, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.8046875, |
| "reward/margin": 1.703125, |
| "reward/rejected": -3.5, |
| "step": 356 |
| }, |
| { |
| "approx. KL/chosen": 20.75, |
| "approx. KL/rejected": 206.0, |
| "epoch": 0.36577868852459017, |
| "grad_norm": 73.65782928466797, |
| "learning_rate": 6.571902617285456e-06, |
| "logp/chosen": -980.0, |
| "logp/rejected": -1004.0, |
| "loss": 1.28662109375, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.875, |
| "reward/margin": 2.125, |
| "reward/rejected": -3.0, |
| "step": 357 |
| }, |
| { |
| "approx. KL/chosen": 52.25, |
| "approx. KL/rejected": 221.0, |
| "epoch": 0.3668032786885246, |
| "grad_norm": 99.26129150390625, |
| "learning_rate": 6.561615819133592e-06, |
| "logp/chosen": -996.0, |
| "logp/rejected": -1224.0, |
| "loss": 1.3544921875, |
| "nll_loss": 0.486328125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.953125, |
| "reward/margin": 1.546875, |
| "reward/rejected": -2.5, |
| "step": 358 |
| }, |
| { |
| "approx. KL/chosen": 13.9375, |
| "approx. KL/rejected": 358.0, |
| "epoch": 0.367827868852459, |
| "grad_norm": 28.62688636779785, |
| "learning_rate": 6.551301345909682e-06, |
| "logp/chosen": -536.0, |
| "logp/rejected": -684.0, |
| "loss": 0.78466796875, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.59375, |
| "reward/margin": 3.03125, |
| "reward/rejected": -3.625, |
| "step": 359 |
| }, |
| { |
| "approx. KL/chosen": 47.75, |
| "approx. KL/rejected": 119.0, |
| "epoch": 0.36885245901639346, |
| "grad_norm": 88.2785873413086, |
| "learning_rate": 6.54095932966947e-06, |
| "logp/chosen": -684.0, |
| "logp/rejected": -1048.0, |
| "loss": 1.06689453125, |
| "nll_loss": 0.466796875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.5546875, |
| "reward/margin": 0.6953125, |
| "reward/rejected": -2.25, |
| "step": 360 |
| }, |
| { |
| "approx. KL/chosen": 26.0, |
| "approx. KL/rejected": 640.0, |
| "epoch": 0.3698770491803279, |
| "grad_norm": 27.099889755249023, |
| "learning_rate": 6.530589902821335e-06, |
| "logp/chosen": -1328.0, |
| "logp/rejected": -1544.0, |
| "loss": 0.864990234375, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.125, |
| "reward/margin": 5.78125, |
| "reward/rejected": -5.65625, |
| "step": 361 |
| }, |
| { |
| "approx. KL/chosen": 85.0, |
| "approx. KL/rejected": 168.0, |
| "epoch": 0.3709016393442623, |
| "grad_norm": 131.75059509277344, |
| "learning_rate": 6.520193198124591e-06, |
| "logp/chosen": -1440.0, |
| "logp/rejected": -1344.0, |
| "loss": 1.57421875, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.421875, |
| "reward/margin": 1.765625, |
| "reward/rejected": -3.1875, |
| "step": 362 |
| }, |
| { |
| "approx. KL/chosen": 167.0, |
| "approx. KL/rejected": 478.0, |
| "epoch": 0.3719262295081967, |
| "grad_norm": 100.37835693359375, |
| "learning_rate": 6.509769348687791e-06, |
| "logp/chosen": -1056.0, |
| "logp/rejected": -1688.0, |
| "loss": 1.4609375, |
| "nll_loss": 0.5078125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -2.125, |
| "reward/margin": 2.109375, |
| "reward/rejected": -4.21875, |
| "step": 363 |
| }, |
| { |
| "approx. KL/chosen": 44.25, |
| "approx. KL/rejected": 175.0, |
| "epoch": 0.3729508196721312, |
| "grad_norm": 39.44411849975586, |
| "learning_rate": 6.499318487967018e-06, |
| "logp/chosen": -728.0, |
| "logp/rejected": -760.0, |
| "loss": 0.9931640625, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.4765625, |
| "reward/margin": 1.578125, |
| "reward/rejected": -3.0625, |
| "step": 364 |
| }, |
| { |
| "approx. KL/chosen": 51.5, |
| "approx. KL/rejected": 560.0, |
| "epoch": 0.3739754098360656, |
| "grad_norm": 45.93659973144531, |
| "learning_rate": 6.488840749764185e-06, |
| "logp/chosen": -972.0, |
| "logp/rejected": -1552.0, |
| "loss": 0.81689453125, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -1.53125, |
| "reward/margin": 3.28125, |
| "reward/rejected": -4.8125, |
| "step": 365 |
| }, |
| { |
| "approx. KL/chosen": 55.5, |
| "approx. KL/rejected": 324.0, |
| "epoch": 0.375, |
| "grad_norm": 42.52275466918945, |
| "learning_rate": 6.478336268225308e-06, |
| "logp/chosen": -640.0, |
| "logp/rejected": -1012.0, |
| "loss": 0.975341796875, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.703125, |
| "reward/margin": 2.53125, |
| "reward/rejected": -4.21875, |
| "step": 366 |
| }, |
| { |
| "approx. KL/chosen": 31.25, |
| "approx. KL/rejected": 190.0, |
| "epoch": 0.3760245901639344, |
| "grad_norm": 96.97118377685547, |
| "learning_rate": 6.467805177838803e-06, |
| "logp/chosen": -908.0, |
| "logp/rejected": -1168.0, |
| "loss": 1.570068359375, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.0546875, |
| "reward/margin": 1.5234375, |
| "reward/rejected": -2.578125, |
| "step": 367 |
| }, |
| { |
| "approx. KL/chosen": 19.25, |
| "approx. KL/rejected": 442.0, |
| "epoch": 0.3770491803278688, |
| "grad_norm": 33.07258605957031, |
| "learning_rate": 6.457247613433757e-06, |
| "logp/chosen": -908.0, |
| "logp/rejected": -1072.0, |
| "loss": 0.8095703125, |
| "nll_loss": 0.6484375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.75, |
| "reward/margin": 4.21875, |
| "reward/rejected": -4.96875, |
| "step": 368 |
| }, |
| { |
| "approx. KL/chosen": 87.0, |
| "approx. KL/rejected": 424.0, |
| "epoch": 0.3780737704918033, |
| "grad_norm": 90.8613510131836, |
| "learning_rate": 6.4466637101782e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1536.0, |
| "loss": 1.4833984375, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -1.5546875, |
| "reward/margin": 3.359375, |
| "reward/rejected": -4.90625, |
| "step": 369 |
| }, |
| { |
| "approx. KL/chosen": 77.5, |
| "approx. KL/rejected": 466.0, |
| "epoch": 0.3790983606557377, |
| "grad_norm": 86.76979064941406, |
| "learning_rate": 6.436053603577379e-06, |
| "logp/chosen": -1304.0, |
| "logp/rejected": -1568.0, |
| "loss": 1.1865234375, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.439453125, |
| "reward/margin": 3.859375, |
| "reward/rejected": -4.28125, |
| "step": 370 |
| }, |
| { |
| "approx. KL/chosen": 86.0, |
| "approx. KL/rejected": 346.0, |
| "epoch": 0.3801229508196721, |
| "grad_norm": 157.61422729492188, |
| "learning_rate": 6.425417429472022e-06, |
| "logp/chosen": -1416.0, |
| "logp/rejected": -1504.0, |
| "loss": 2.29833984375, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.6015625, |
| "reward/margin": 1.453125, |
| "reward/rejected": -3.046875, |
| "step": 371 |
| }, |
| { |
| "approx. KL/chosen": 17.0, |
| "approx. KL/rejected": 468.0, |
| "epoch": 0.38114754098360654, |
| "grad_norm": 11.884814262390137, |
| "learning_rate": 6.414755324036597e-06, |
| "logp/chosen": -848.0, |
| "logp/rejected": -860.0, |
| "loss": 0.63232421875, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.5, |
| "reward/margin": 4.21875, |
| "reward/rejected": -4.71875, |
| "step": 372 |
| }, |
| { |
| "approx. KL/chosen": 57.75, |
| "approx. KL/rejected": 580.0, |
| "epoch": 0.382172131147541, |
| "grad_norm": 48.06128692626953, |
| "learning_rate": 6.40406742377757e-06, |
| "logp/chosen": -728.0, |
| "logp/rejected": -1072.0, |
| "loss": 1.26806640625, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.75, |
| "reward/margin": 3.703125, |
| "reward/rejected": -4.46875, |
| "step": 373 |
| }, |
| { |
| "approx. KL/chosen": 58.5, |
| "approx. KL/rejected": 412.0, |
| "epoch": 0.3831967213114754, |
| "grad_norm": 68.96261596679688, |
| "learning_rate": 6.393353865531656e-06, |
| "logp/chosen": -1000.0, |
| "logp/rejected": -1400.0, |
| "loss": 1.002685546875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.57421875, |
| "reward/margin": 3.21875, |
| "reward/rejected": -3.796875, |
| "step": 374 |
| }, |
| { |
| "approx. KL/chosen": 34.25, |
| "approx. KL/rejected": 532.0, |
| "epoch": 0.38422131147540983, |
| "grad_norm": 24.105201721191406, |
| "learning_rate": 6.382614786464069e-06, |
| "logp/chosen": -896.0, |
| "logp/rejected": -1472.0, |
| "loss": 0.766845703125, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.050048828125, |
| "reward/margin": 5.34375, |
| "reward/rejected": -5.3125, |
| "step": 375 |
| }, |
| { |
| "approx. KL/chosen": 93.5, |
| "approx. KL/rejected": 144.0, |
| "epoch": 0.38524590163934425, |
| "grad_norm": 104.90898895263672, |
| "learning_rate": 6.371850324066767e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -936.0, |
| "loss": 1.645263671875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.9765625, |
| "reward/margin": 0.69921875, |
| "reward/rejected": -2.671875, |
| "step": 376 |
| }, |
| { |
| "approx. KL/chosen": 41.25, |
| "approx. KL/rejected": 1064.0, |
| "epoch": 0.3862704918032787, |
| "grad_norm": 74.27389526367188, |
| "learning_rate": 6.361060616156685e-06, |
| "logp/chosen": -1144.0, |
| "logp/rejected": -1592.0, |
| "loss": 1.07421875, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.94921875, |
| "reward/margin": 5.75, |
| "reward/rejected": -6.71875, |
| "step": 377 |
| }, |
| { |
| "approx. KL/chosen": 13.8125, |
| "approx. KL/rejected": 944.0, |
| "epoch": 0.38729508196721313, |
| "grad_norm": 25.533950805664062, |
| "learning_rate": 6.3502458008739795e-06, |
| "logp/chosen": -680.0, |
| "logp/rejected": -1256.0, |
| "loss": 0.755859375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.2255859375, |
| "reward/margin": 6.25, |
| "reward/rejected": -6.46875, |
| "step": 378 |
| }, |
| { |
| "approx. KL/chosen": 45.0, |
| "approx. KL/rejected": 96.0, |
| "epoch": 0.38831967213114754, |
| "grad_norm": 103.07498931884766, |
| "learning_rate": 6.3394060166802535e-06, |
| "logp/chosen": -1192.0, |
| "logp/rejected": -1256.0, |
| "loss": 1.5595703125, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -1.703125, |
| "reward/margin": 0.349609375, |
| "reward/rejected": -2.046875, |
| "step": 379 |
| }, |
| { |
| "approx. KL/chosen": 27.75, |
| "approx. KL/rejected": 478.0, |
| "epoch": 0.38934426229508196, |
| "grad_norm": 17.405990600585938, |
| "learning_rate": 6.328541402356784e-06, |
| "logp/chosen": -760.0, |
| "logp/rejected": -1096.0, |
| "loss": 0.68212890625, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.0751953125, |
| "reward/margin": 5.0, |
| "reward/rejected": -5.09375, |
| "step": 380 |
| }, |
| { |
| "approx. KL/chosen": 37.25, |
| "approx. KL/rejected": 260.0, |
| "epoch": 0.39036885245901637, |
| "grad_norm": 76.76506042480469, |
| "learning_rate": 6.317652097002753e-06, |
| "logp/chosen": -900.0, |
| "logp/rejected": -936.0, |
| "loss": 1.41650390625, |
| "nll_loss": 0.671875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.1484375, |
| "reward/margin": 1.6953125, |
| "reward/rejected": -2.84375, |
| "step": 381 |
| }, |
| { |
| "approx. KL/chosen": 64.0, |
| "approx. KL/rejected": 310.0, |
| "epoch": 0.39139344262295084, |
| "grad_norm": 63.4621696472168, |
| "learning_rate": 6.3067382400334535e-06, |
| "logp/chosen": -744.0, |
| "logp/rejected": -996.0, |
| "loss": 1.01953125, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 3.40625, |
| "reward/rejected": -4.25, |
| "step": 382 |
| }, |
| { |
| "approx. KL/chosen": 168.0, |
| "approx. KL/rejected": 48.5, |
| "epoch": 0.39241803278688525, |
| "grad_norm": 142.26358032226562, |
| "learning_rate": 6.295799971178518e-06, |
| "logp/chosen": -1056.0, |
| "logp/rejected": -752.0, |
| "loss": 2.380126953125, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.75, |
| "reward/margin": -0.2734375, |
| "reward/rejected": -1.4765625, |
| "step": 383 |
| }, |
| { |
| "approx. KL/chosen": 49.5, |
| "approx. KL/rejected": 442.0, |
| "epoch": 0.39344262295081966, |
| "grad_norm": 24.155744552612305, |
| "learning_rate": 6.2848374304801194e-06, |
| "logp/chosen": -960.0, |
| "logp/rejected": -1296.0, |
| "loss": 0.796630859375, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -1.0, |
| "reward/margin": 3.609375, |
| "reward/rejected": -4.59375, |
| "step": 384 |
| }, |
| { |
| "approx. KL/chosen": 180.0, |
| "approx. KL/rejected": 560.0, |
| "epoch": 0.3944672131147541, |
| "grad_norm": 99.10308837890625, |
| "learning_rate": 6.273850758291182e-06, |
| "logp/chosen": -1336.0, |
| "logp/rejected": -1488.0, |
| "loss": 1.52490234375, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.65625, |
| "reward/margin": 4.15625, |
| "reward/rejected": -5.8125, |
| "step": 385 |
| }, |
| { |
| "approx. KL/chosen": 30.0, |
| "approx. KL/rejected": 480.0, |
| "epoch": 0.39549180327868855, |
| "grad_norm": 45.18642044067383, |
| "learning_rate": 6.262840095273586e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -1020.0, |
| "loss": 0.740478515625, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 3.90625, |
| "reward/rejected": -4.5, |
| "step": 386 |
| }, |
| { |
| "approx. KL/chosen": 173.0, |
| "approx. KL/rejected": 76.0, |
| "epoch": 0.39651639344262296, |
| "grad_norm": 164.05628967285156, |
| "learning_rate": 6.251805582396367e-06, |
| "logp/chosen": -768.0, |
| "logp/rejected": -812.0, |
| "loss": 2.5556640625, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -2.578125, |
| "reward/margin": -1.046875, |
| "reward/rejected": -1.5390625, |
| "step": 387 |
| }, |
| { |
| "approx. KL/chosen": 151.0, |
| "approx. KL/rejected": 199.0, |
| "epoch": 0.3975409836065574, |
| "grad_norm": 81.71810913085938, |
| "learning_rate": 6.240747360933904e-06, |
| "logp/chosen": -636.0, |
| "logp/rejected": -772.0, |
| "loss": 1.52587890625, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.5234375, |
| "reward/margin": 1.15625, |
| "reward/rejected": -2.6875, |
| "step": 388 |
| }, |
| { |
| "approx. KL/chosen": 93.5, |
| "approx. KL/rejected": 136.0, |
| "epoch": 0.3985655737704918, |
| "grad_norm": 106.06842803955078, |
| "learning_rate": 6.229665572464119e-06, |
| "logp/chosen": -1096.0, |
| "logp/rejected": -1200.0, |
| "loss": 1.3134765625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.6796875, |
| "reward/margin": 0.890625, |
| "reward/rejected": -2.578125, |
| "step": 389 |
| }, |
| { |
| "approx. KL/chosen": 13.3125, |
| "approx. KL/rejected": 460.0, |
| "epoch": 0.39959016393442626, |
| "grad_norm": 24.24275016784668, |
| "learning_rate": 6.218560358866663e-06, |
| "logp/chosen": -1176.0, |
| "logp/rejected": -1208.0, |
| "loss": 0.615478515625, |
| "nll_loss": 0.478515625, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.62890625, |
| "reward/margin": 4.125, |
| "reward/rejected": -4.78125, |
| "step": 390 |
| }, |
| { |
| "approx. KL/chosen": 26.0, |
| "approx. KL/rejected": 752.0, |
| "epoch": 0.40061475409836067, |
| "grad_norm": 84.49885559082031, |
| "learning_rate": 6.207431862321096e-06, |
| "logp/chosen": -1064.0, |
| "logp/rejected": -1832.0, |
| "loss": 0.934326171875, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 5.59375, |
| "reward/rejected": -6.0, |
| "step": 391 |
| }, |
| { |
| "epoch": 0.40061475409836067, |
| "eval_approx. KL/chosen": 59.79, |
| "eval_approx. KL/rejected": 396.3, |
| "eval_logp/chosen": -969.36, |
| "eval_logp/rejected": -1296.8, |
| "eval_loss": 1.2786718606948853, |
| "eval_nll_loss": 0.555546875, |
| "eval_reward/accuracy": 0.69, |
| "eval_reward/chosen": -0.9362890625, |
| "eval_reward/margin": 3.06642578125, |
| "eval_reward/rejected": -4.0034375, |
| "eval_runtime": 46.7015, |
| "eval_samples_per_second": 4.24, |
| "eval_steps_per_second": 0.535, |
| "step": 391 |
| }, |
| { |
| "approx. KL/chosen": 36.75, |
| "approx. KL/rejected": 233.0, |
| "epoch": 0.4016393442622951, |
| "grad_norm": 72.73355102539062, |
| "learning_rate": 6.196280225305068e-06, |
| "logp/chosen": -892.0, |
| "logp/rejected": -1248.0, |
| "loss": 1.287841796875, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.44921875, |
| "reward/margin": 2.453125, |
| "reward/rejected": -2.90625, |
| "step": 392 |
| }, |
| { |
| "approx. KL/chosen": 86.5, |
| "approx. KL/rejected": 249.0, |
| "epoch": 0.4026639344262295, |
| "grad_norm": 22.260334014892578, |
| "learning_rate": 6.185105590592496e-06, |
| "logp/chosen": -680.0, |
| "logp/rejected": -1136.0, |
| "loss": 0.6053466796875, |
| "nll_loss": 0.474609375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.07470703125, |
| "reward/margin": 3.59375, |
| "reward/rejected": -3.515625, |
| "step": 393 |
| }, |
| { |
| "approx. KL/chosen": 45.25, |
| "approx. KL/rejected": 245.0, |
| "epoch": 0.4036885245901639, |
| "grad_norm": 85.36290740966797, |
| "learning_rate": 6.1739081012517364e-06, |
| "logp/chosen": -972.0, |
| "logp/rejected": -988.0, |
| "loss": 1.251220703125, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.953125, |
| "reward/margin": 2.6875, |
| "reward/rejected": -3.640625, |
| "step": 394 |
| }, |
| { |
| "approx. KL/chosen": 51.0, |
| "approx. KL/rejected": 219.0, |
| "epoch": 0.4047131147540984, |
| "grad_norm": 77.00773620605469, |
| "learning_rate": 6.16268790064375e-06, |
| "logp/chosen": -968.0, |
| "logp/rejected": -1384.0, |
| "loss": 0.9130859375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.08740234375, |
| "reward/margin": 3.359375, |
| "reward/rejected": -3.28125, |
| "step": 395 |
| }, |
| { |
| "approx. KL/chosen": 62.0, |
| "approx. KL/rejected": 326.0, |
| "epoch": 0.4057377049180328, |
| "grad_norm": 58.21205139160156, |
| "learning_rate": 6.151445132420272e-06, |
| "logp/chosen": -872.0, |
| "logp/rejected": -1128.0, |
| "loss": 1.50830078125, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0, |
| "reward/margin": 2.5, |
| "reward/rejected": -3.5, |
| "step": 396 |
| }, |
| { |
| "approx. KL/chosen": 56.25, |
| "approx. KL/rejected": 140.0, |
| "epoch": 0.4067622950819672, |
| "grad_norm": 143.65475463867188, |
| "learning_rate": 6.140179940521969e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -896.0, |
| "loss": 2.345703125, |
| "nll_loss": 0.703125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.2265625, |
| "reward/margin": 0.3515625, |
| "reward/rejected": -1.578125, |
| "step": 397 |
| }, |
| { |
| "approx. KL/chosen": 12.5625, |
| "approx. KL/rejected": 476.0, |
| "epoch": 0.4077868852459016, |
| "grad_norm": 39.2029914855957, |
| "learning_rate": 6.128892469176596e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -1400.0, |
| "loss": 0.96484375, |
| "nll_loss": 0.71484375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.67578125, |
| "reward/margin": 4.1875, |
| "reward/rejected": -4.84375, |
| "step": 398 |
| }, |
| { |
| "approx. KL/chosen": 78.0, |
| "approx. KL/rejected": 249.0, |
| "epoch": 0.4088114754098361, |
| "grad_norm": 35.92908477783203, |
| "learning_rate": 6.117582862897149e-06, |
| "logp/chosen": -1144.0, |
| "logp/rejected": -1776.0, |
| "loss": 0.73974609375, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.6015625, |
| "reward/margin": 4.5, |
| "reward/rejected": -3.90625, |
| "step": 399 |
| }, |
| { |
| "approx. KL/chosen": 92.0, |
| "approx. KL/rejected": 314.0, |
| "epoch": 0.4098360655737705, |
| "grad_norm": 82.80913543701172, |
| "learning_rate": 6.106251266480021e-06, |
| "logp/chosen": -864.0, |
| "logp/rejected": -1096.0, |
| "loss": 1.82373046875, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.953125, |
| "reward/margin": 2.765625, |
| "reward/rejected": -3.71875, |
| "step": 400 |
| }, |
| { |
| "approx. KL/chosen": 72.0, |
| "approx. KL/rejected": 468.0, |
| "epoch": 0.4108606557377049, |
| "grad_norm": 70.79622650146484, |
| "learning_rate": 6.094897825003139e-06, |
| "logp/chosen": -1320.0, |
| "logp/rejected": -1352.0, |
| "loss": 0.67529296875, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -1.203125, |
| "reward/margin": 4.03125, |
| "reward/rejected": -5.25, |
| "step": 401 |
| }, |
| { |
| "approx. KL/chosen": 30.0, |
| "approx. KL/rejected": 34.25, |
| "epoch": 0.41188524590163933, |
| "grad_norm": 108.22535705566406, |
| "learning_rate": 6.083522683824117e-06, |
| "logp/chosen": -884.0, |
| "logp/rejected": -820.0, |
| "loss": 1.70751953125, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 0.076171875, |
| "reward/rejected": -0.875, |
| "step": 402 |
| }, |
| { |
| "approx. KL/chosen": 97.5, |
| "approx. KL/rejected": 135.0, |
| "epoch": 0.41290983606557374, |
| "grad_norm": 144.07086181640625, |
| "learning_rate": 6.072125988578385e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1304.0, |
| "loss": 2.09423828125, |
| "nll_loss": 0.65625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.1875, |
| "reward/margin": 0.7890625, |
| "reward/rejected": -1.9765625, |
| "step": 403 |
| }, |
| { |
| "approx. KL/chosen": 18.5, |
| "approx. KL/rejected": 39.5, |
| "epoch": 0.4139344262295082, |
| "grad_norm": 73.78527069091797, |
| "learning_rate": 6.060707885177328e-06, |
| "logp/chosen": -468.0, |
| "logp/rejected": -508.0, |
| "loss": 1.1083984375, |
| "nll_loss": 0.455078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.000244140625, |
| "reward/margin": 1.15625, |
| "reward/rejected": -1.15625, |
| "step": 404 |
| }, |
| { |
| "approx. KL/chosen": 84.0, |
| "approx. KL/rejected": 139.0, |
| "epoch": 0.4149590163934426, |
| "grad_norm": 142.4896240234375, |
| "learning_rate": 6.0492685198064225e-06, |
| "logp/chosen": -972.0, |
| "logp/rejected": -1020.0, |
| "loss": 1.773681640625, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.4296875, |
| "reward/margin": 0.4765625, |
| "reward/rejected": -1.8984375, |
| "step": 405 |
| }, |
| { |
| "approx. KL/chosen": 22.25, |
| "approx. KL/rejected": 163.0, |
| "epoch": 0.41598360655737704, |
| "grad_norm": 40.44302749633789, |
| "learning_rate": 6.037808038923359e-06, |
| "logp/chosen": -980.0, |
| "logp/rejected": -1112.0, |
| "loss": 0.804443359375, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 2.421875, |
| "reward/rejected": -2.578125, |
| "step": 406 |
| }, |
| { |
| "approx. KL/chosen": 104.5, |
| "approx. KL/rejected": 20.0, |
| "epoch": 0.41700819672131145, |
| "grad_norm": 113.4881362915039, |
| "learning_rate": 6.0263265892561665e-06, |
| "logp/chosen": -880.0, |
| "logp/rejected": -1032.0, |
| "loss": 1.99267578125, |
| "nll_loss": 0.5, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.0, |
| "reward/margin": -0.42578125, |
| "reward/rejected": -0.57421875, |
| "step": 407 |
| }, |
| { |
| "approx. KL/chosen": 46.25, |
| "approx. KL/rejected": 484.0, |
| "epoch": 0.4180327868852459, |
| "grad_norm": 50.905208587646484, |
| "learning_rate": 6.014824317801343e-06, |
| "logp/chosen": -912.0, |
| "logp/rejected": -1416.0, |
| "loss": 0.7939453125, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 4.40625, |
| "reward/rejected": -4.9375, |
| "step": 408 |
| }, |
| { |
| "approx. KL/chosen": 81.0, |
| "approx. KL/rejected": 444.0, |
| "epoch": 0.41905737704918034, |
| "grad_norm": 16.74032211303711, |
| "learning_rate": 6.00330137182196e-06, |
| "logp/chosen": -1112.0, |
| "logp/rejected": -1808.0, |
| "loss": 0.7236328125, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -1.25, |
| "reward/margin": 3.859375, |
| "reward/rejected": -5.125, |
| "step": 409 |
| }, |
| { |
| "approx. KL/chosen": 24.5, |
| "approx. KL/rejected": 460.0, |
| "epoch": 0.42008196721311475, |
| "grad_norm": 22.307126998901367, |
| "learning_rate": 5.991757898845788e-06, |
| "logp/chosen": -844.0, |
| "logp/rejected": -1304.0, |
| "loss": 0.740966796875, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.000244140625, |
| "reward/margin": 4.90625, |
| "reward/rejected": -4.90625, |
| "step": 410 |
| }, |
| { |
| "approx. KL/chosen": 22.0, |
| "approx. KL/rejected": 81.0, |
| "epoch": 0.42110655737704916, |
| "grad_norm": 96.17124938964844, |
| "learning_rate": 5.980194046663403e-06, |
| "logp/chosen": -640.0, |
| "logp/rejected": -964.0, |
| "loss": 1.37744140625, |
| "nll_loss": 0.443359375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.53125, |
| "reward/margin": 0.8046875, |
| "reward/rejected": -1.3359375, |
| "step": 411 |
| }, |
| { |
| "approx. KL/chosen": 149.0, |
| "approx. KL/rejected": 68.0, |
| "epoch": 0.42213114754098363, |
| "grad_norm": 155.61643981933594, |
| "learning_rate": 5.968609963326293e-06, |
| "logp/chosen": -1424.0, |
| "logp/rejected": -1272.0, |
| "loss": 2.2998046875, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.8984375, |
| "reward/margin": -0.1015625, |
| "reward/rejected": -1.8046875, |
| "step": 412 |
| }, |
| { |
| "approx. KL/chosen": 30.0, |
| "approx. KL/rejected": 90.0, |
| "epoch": 0.42315573770491804, |
| "grad_norm": 88.52623748779297, |
| "learning_rate": 5.957005797144969e-06, |
| "logp/chosen": -980.0, |
| "logp/rejected": -932.0, |
| "loss": 1.184814453125, |
| "nll_loss": 0.5078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 1.203125, |
| "reward/rejected": -2.0, |
| "step": 413 |
| }, |
| { |
| "approx. KL/chosen": 23.25, |
| "approx. KL/rejected": 256.0, |
| "epoch": 0.42418032786885246, |
| "grad_norm": 73.61592102050781, |
| "learning_rate": 5.945381696687058e-06, |
| "logp/chosen": -880.0, |
| "logp/rejected": -1040.0, |
| "loss": 1.109375, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 3.140625, |
| "reward/rejected": -3.6875, |
| "step": 414 |
| }, |
| { |
| "approx. KL/chosen": 28.0, |
| "approx. KL/rejected": 103.5, |
| "epoch": 0.42520491803278687, |
| "grad_norm": 66.11711883544922, |
| "learning_rate": 5.933737810775404e-06, |
| "logp/chosen": -784.0, |
| "logp/rejected": -736.0, |
| "loss": 0.95947265625, |
| "nll_loss": 0.5078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.7265625, |
| "reward/margin": 1.4375, |
| "reward/rejected": -2.171875, |
| "step": 415 |
| }, |
| { |
| "approx. KL/chosen": 49.25, |
| "approx. KL/rejected": 504.0, |
| "epoch": 0.4262295081967213, |
| "grad_norm": 28.858802795410156, |
| "learning_rate": 5.922074288486167e-06, |
| "logp/chosen": -892.0, |
| "logp/rejected": -1552.0, |
| "loss": 0.701171875, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.1494140625, |
| "reward/margin": 6.03125, |
| "reward/rejected": -5.875, |
| "step": 416 |
| }, |
| { |
| "approx. KL/chosen": 32.0, |
| "approx. KL/rejected": 370.0, |
| "epoch": 0.42725409836065575, |
| "grad_norm": 142.6680145263672, |
| "learning_rate": 5.910391279146908e-06, |
| "logp/chosen": -1184.0, |
| "logp/rejected": -1872.0, |
| "loss": 1.53955078125, |
| "nll_loss": 0.52734375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.203125, |
| "reward/margin": 2.203125, |
| "reward/rejected": -3.40625, |
| "step": 417 |
| }, |
| { |
| "approx. KL/chosen": 41.0, |
| "approx. KL/rejected": 528.0, |
| "epoch": 0.42827868852459017, |
| "grad_norm": 65.89132690429688, |
| "learning_rate": 5.8986889323346805e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -1392.0, |
| "loss": 0.9921875, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.8984375, |
| "reward/margin": 4.3125, |
| "reward/rejected": -5.1875, |
| "step": 418 |
| }, |
| { |
| "approx. KL/chosen": 42.5, |
| "approx. KL/rejected": 442.0, |
| "epoch": 0.4293032786885246, |
| "grad_norm": 19.042774200439453, |
| "learning_rate": 5.886967397874116e-06, |
| "logp/chosen": -924.0, |
| "logp/rejected": -1400.0, |
| "loss": 0.69970703125, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.17578125, |
| "reward/margin": 5.03125, |
| "reward/rejected": -4.84375, |
| "step": 419 |
| }, |
| { |
| "approx. KL/chosen": 65.5, |
| "approx. KL/rejected": 211.0, |
| "epoch": 0.430327868852459, |
| "grad_norm": 95.75060272216797, |
| "learning_rate": 5.875226825835503e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -996.0, |
| "loss": 1.3232421875, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.1005859375, |
| "reward/margin": 3.0625, |
| "reward/rejected": -2.953125, |
| "step": 420 |
| }, |
| { |
| "approx. KL/chosen": 28.25, |
| "approx. KL/rejected": 320.0, |
| "epoch": 0.43135245901639346, |
| "grad_norm": 36.977352142333984, |
| "learning_rate": 5.863467366532867e-06, |
| "logp/chosen": -748.0, |
| "logp/rejected": -1032.0, |
| "loss": 0.91259765625, |
| "nll_loss": 0.52734375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.275390625, |
| "reward/margin": 3.40625, |
| "reward/rejected": -3.125, |
| "step": 421 |
| }, |
| { |
| "approx. KL/chosen": 6.0625, |
| "approx. KL/rejected": 241.0, |
| "epoch": 0.4323770491803279, |
| "grad_norm": 114.46843719482422, |
| "learning_rate": 5.851689170522048e-06, |
| "logp/chosen": -1248.0, |
| "logp/rejected": -1720.0, |
| "loss": 1.185546875, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.2255859375, |
| "reward/margin": 2.359375, |
| "reward/rejected": -2.578125, |
| "step": 422 |
| }, |
| { |
| "approx. KL/chosen": 50.0, |
| "approx. KL/rejected": 264.0, |
| "epoch": 0.4334016393442623, |
| "grad_norm": 122.7032470703125, |
| "learning_rate": 5.83989238859877e-06, |
| "logp/chosen": -892.0, |
| "logp/rejected": -1448.0, |
| "loss": 2.0986328125, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 2.171875, |
| "reward/rejected": -2.78125, |
| "step": 423 |
| }, |
| { |
| "approx. KL/chosen": 268.0, |
| "approx. KL/rejected": 368.0, |
| "epoch": 0.4344262295081967, |
| "grad_norm": 148.0244598388672, |
| "learning_rate": 5.82807717179671e-06, |
| "logp/chosen": -1392.0, |
| "logp/rejected": -1560.0, |
| "loss": 1.7138671875, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -2.109375, |
| "reward/margin": 2.296875, |
| "reward/rejected": -4.40625, |
| "step": 424 |
| }, |
| { |
| "approx. KL/chosen": 47.0, |
| "approx. KL/rejected": 118.0, |
| "epoch": 0.4354508196721312, |
| "grad_norm": 120.8546142578125, |
| "learning_rate": 5.816243671385572e-06, |
| "logp/chosen": -852.0, |
| "logp/rejected": -988.0, |
| "loss": 2.12255859375, |
| "nll_loss": 0.64453125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 0.248046875, |
| "reward/rejected": -1.1484375, |
| "step": 425 |
| }, |
| { |
| "approx. KL/chosen": 89.0, |
| "approx. KL/rejected": 290.0, |
| "epoch": 0.4364754098360656, |
| "grad_norm": 66.18994140625, |
| "learning_rate": 5.804392038869138e-06, |
| "logp/chosen": -856.0, |
| "logp/rejected": -1040.0, |
| "loss": 1.57373046875, |
| "nll_loss": 0.6953125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.451171875, |
| "reward/margin": 3.25, |
| "reward/rejected": -3.703125, |
| "step": 426 |
| }, |
| { |
| "approx. KL/chosen": 17.625, |
| "approx. KL/rejected": 274.0, |
| "epoch": 0.4375, |
| "grad_norm": 63.247867584228516, |
| "learning_rate": 5.792522425983335e-06, |
| "logp/chosen": -628.0, |
| "logp/rejected": -824.0, |
| "loss": 0.937255859375, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.58984375, |
| "reward/margin": 2.921875, |
| "reward/rejected": -3.5, |
| "step": 427 |
| }, |
| { |
| "approx. KL/chosen": 82.0, |
| "approx. KL/rejected": 145.0, |
| "epoch": 0.4385245901639344, |
| "grad_norm": 107.6949462890625, |
| "learning_rate": 5.780634984694297e-06, |
| "logp/chosen": -812.0, |
| "logp/rejected": -924.0, |
| "loss": 1.8760986328125, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.4765625, |
| "reward/margin": 0.77734375, |
| "reward/rejected": -2.25, |
| "step": 428 |
| }, |
| { |
| "approx. KL/chosen": 34.5, |
| "approx. KL/rejected": 108.5, |
| "epoch": 0.4395491803278688, |
| "grad_norm": 106.19087219238281, |
| "learning_rate": 5.768729867196407e-06, |
| "logp/chosen": -876.0, |
| "logp/rejected": -924.0, |
| "loss": 1.18310546875, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.875, |
| "reward/margin": 1.5390625, |
| "reward/rejected": -2.421875, |
| "step": 429 |
| }, |
| { |
| "approx. KL/chosen": 87.0, |
| "approx. KL/rejected": 81.5, |
| "epoch": 0.4405737704918033, |
| "grad_norm": 73.81458282470703, |
| "learning_rate": 5.756807225910363e-06, |
| "logp/chosen": -1152.0, |
| "logp/rejected": -1256.0, |
| "loss": 1.14794921875, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.30078125, |
| "reward/margin": 1.9296875, |
| "reward/rejected": -1.625, |
| "step": 430 |
| }, |
| { |
| "approx. KL/chosen": 52.5, |
| "approx. KL/rejected": 344.0, |
| "epoch": 0.4415983606557377, |
| "grad_norm": 85.26643371582031, |
| "learning_rate": 5.744867213481214e-06, |
| "logp/chosen": -1240.0, |
| "logp/rejected": -1104.0, |
| "loss": 1.0166015625, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0, |
| "reward/margin": 2.296875, |
| "reward/rejected": -3.296875, |
| "step": 431 |
| }, |
| { |
| "approx. KL/chosen": 15.3125, |
| "approx. KL/rejected": 316.0, |
| "epoch": 0.4426229508196721, |
| "grad_norm": 16.78167724609375, |
| "learning_rate": 5.732909982776413e-06, |
| "logp/chosen": -424.0, |
| "logp/rejected": -516.0, |
| "loss": 0.65625, |
| "nll_loss": 0.5078125, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.326171875, |
| "reward/margin": 3.109375, |
| "reward/rejected": -3.421875, |
| "step": 432 |
| }, |
| { |
| "approx. KL/chosen": 30.625, |
| "approx. KL/rejected": 96.0, |
| "epoch": 0.44364754098360654, |
| "grad_norm": 70.34516143798828, |
| "learning_rate": 5.7209356868838575e-06, |
| "logp/chosen": -572.0, |
| "logp/rejected": -732.0, |
| "loss": 1.44384765625, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 0.65234375, |
| "reward/rejected": -1.75, |
| "step": 433 |
| }, |
| { |
| "approx. KL/chosen": 28.5, |
| "approx. KL/rejected": 436.0, |
| "epoch": 0.444672131147541, |
| "grad_norm": 46.55507278442383, |
| "learning_rate": 5.708944479109932e-06, |
| "logp/chosen": -668.0, |
| "logp/rejected": -1088.0, |
| "loss": 0.9345703125, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.5, |
| "reward/margin": 4.40625, |
| "reward/rejected": -4.90625, |
| "step": 434 |
| }, |
| { |
| "approx. KL/chosen": 30.75, |
| "approx. KL/rejected": 143.0, |
| "epoch": 0.4456967213114754, |
| "grad_norm": 55.545982360839844, |
| "learning_rate": 5.696936512977537e-06, |
| "logp/chosen": -592.0, |
| "logp/rejected": -760.0, |
| "loss": 0.99072265625, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0546875, |
| "reward/margin": 1.8046875, |
| "reward/rejected": -2.859375, |
| "step": 435 |
| }, |
| { |
| "approx. KL/chosen": 62.5, |
| "approx. KL/rejected": 772.0, |
| "epoch": 0.44672131147540983, |
| "grad_norm": 39.814208984375, |
| "learning_rate": 5.684911942224138e-06, |
| "logp/chosen": -972.0, |
| "logp/rejected": -2208.0, |
| "loss": 1.1767578125, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.1025390625, |
| "reward/margin": 6.59375, |
| "reward/rejected": -6.5, |
| "step": 436 |
| }, |
| { |
| "approx. KL/chosen": 29.25, |
| "approx. KL/rejected": 402.0, |
| "epoch": 0.44774590163934425, |
| "grad_norm": 80.66665649414062, |
| "learning_rate": 5.672870920799781e-06, |
| "logp/chosen": -844.0, |
| "logp/rejected": -1080.0, |
| "loss": 1.1796875, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 3.328125, |
| "reward/rejected": -4.1875, |
| "step": 437 |
| }, |
| { |
| "approx. KL/chosen": 31.0, |
| "approx. KL/rejected": 129.0, |
| "epoch": 0.4487704918032787, |
| "grad_norm": 38.35740280151367, |
| "learning_rate": 5.660813602865133e-06, |
| "logp/chosen": -628.0, |
| "logp/rejected": -552.0, |
| "loss": 0.94873046875, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.375, |
| "reward/margin": 2.3125, |
| "reward/rejected": -2.6875, |
| "step": 438 |
| }, |
| { |
| "approx. KL/chosen": 114.0, |
| "approx. KL/rejected": 492.0, |
| "epoch": 0.44979508196721313, |
| "grad_norm": 107.0799789428711, |
| "learning_rate": 5.648740142789507e-06, |
| "logp/chosen": -1504.0, |
| "logp/rejected": -1872.0, |
| "loss": 1.58154296875, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -2.0, |
| "reward/margin": 3.3125, |
| "reward/rejected": -5.3125, |
| "step": 439 |
| }, |
| { |
| "approx. KL/chosen": 30.25, |
| "approx. KL/rejected": 47.5, |
| "epoch": 0.45081967213114754, |
| "grad_norm": 67.17232513427734, |
| "learning_rate": 5.6366506951488755e-06, |
| "logp/chosen": -912.0, |
| "logp/rejected": -932.0, |
| "loss": 1.26318359375, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.25, |
| "reward/margin": 1.28125, |
| "reward/rejected": -1.53125, |
| "step": 440 |
| }, |
| { |
| "approx. KL/chosen": 26.25, |
| "approx. KL/rejected": 720.0, |
| "epoch": 0.45184426229508196, |
| "grad_norm": 76.6970443725586, |
| "learning_rate": 5.624545414723908e-06, |
| "logp/chosen": -848.0, |
| "logp/rejected": -1352.0, |
| "loss": 1.1435546875, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.953125, |
| "reward/margin": 5.125, |
| "reward/rejected": -6.09375, |
| "step": 441 |
| }, |
| { |
| "approx. KL/chosen": 22.625, |
| "approx. KL/rejected": 141.0, |
| "epoch": 0.45286885245901637, |
| "grad_norm": 87.95294189453125, |
| "learning_rate": 5.6124244564979745e-06, |
| "logp/chosen": -572.0, |
| "logp/rejected": -936.0, |
| "loss": 1.29833984375, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 1.8046875, |
| "reward/rejected": -2.453125, |
| "step": 442 |
| }, |
| { |
| "approx. KL/chosen": 22.25, |
| "approx. KL/rejected": 206.0, |
| "epoch": 0.45389344262295084, |
| "grad_norm": 52.289207458496094, |
| "learning_rate": 5.600287975655167e-06, |
| "logp/chosen": -856.0, |
| "logp/rejected": -1032.0, |
| "loss": 1.02490234375, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.67578125, |
| "reward/margin": 2.125, |
| "reward/rejected": -2.796875, |
| "step": 443 |
| }, |
| { |
| "approx. KL/chosen": 8.25, |
| "approx. KL/rejected": 482.0, |
| "epoch": 0.45491803278688525, |
| "grad_norm": 21.893890380859375, |
| "learning_rate": 5.588136127578317e-06, |
| "logp/chosen": -652.0, |
| "logp/rejected": -916.0, |
| "loss": 0.653564453125, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.0498046875, |
| "reward/margin": 5.03125, |
| "reward/rejected": -5.0, |
| "step": 444 |
| }, |
| { |
| "approx. KL/chosen": 112.5, |
| "approx. KL/rejected": 430.0, |
| "epoch": 0.45594262295081966, |
| "grad_norm": 95.07337188720703, |
| "learning_rate": 5.575969067846999e-06, |
| "logp/chosen": -960.0, |
| "logp/rejected": -1432.0, |
| "loss": 1.4677734375, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.8828125, |
| "reward/margin": 2.5, |
| "reward/rejected": -4.375, |
| "step": 445 |
| }, |
| { |
| "approx. KL/chosen": 39.25, |
| "approx. KL/rejected": 664.0, |
| "epoch": 0.4569672131147541, |
| "grad_norm": 36.29835891723633, |
| "learning_rate": 5.563786952235544e-06, |
| "logp/chosen": -1096.0, |
| "logp/rejected": -1472.0, |
| "loss": 0.79541015625, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.04931640625, |
| "reward/margin": 5.9375, |
| "reward/rejected": -6.0, |
| "step": 446 |
| }, |
| { |
| "approx. KL/chosen": 114.5, |
| "approx. KL/rejected": 362.0, |
| "epoch": 0.45799180327868855, |
| "grad_norm": 87.63541412353516, |
| "learning_rate": 5.551589936711039e-06, |
| "logp/chosen": -820.0, |
| "logp/rejected": -1152.0, |
| "loss": 2.03125, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.7265625, |
| "reward/margin": 1.890625, |
| "reward/rejected": -3.609375, |
| "step": 447 |
| }, |
| { |
| "approx. KL/chosen": 32.25, |
| "approx. KL/rejected": 244.0, |
| "epoch": 0.45901639344262296, |
| "grad_norm": 34.155731201171875, |
| "learning_rate": 5.539378177431336e-06, |
| "logp/chosen": -812.0, |
| "logp/rejected": -1336.0, |
| "loss": 0.764404296875, |
| "nll_loss": 0.4921875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.349609375, |
| "reward/margin": 3.828125, |
| "reward/rejected": -3.484375, |
| "step": 448 |
| }, |
| { |
| "approx. KL/chosen": 110.0, |
| "approx. KL/rejected": 57.5, |
| "epoch": 0.4600409836065574, |
| "grad_norm": 124.01118469238281, |
| "learning_rate": 5.527151830743054e-06, |
| "logp/chosen": -928.0, |
| "logp/rejected": -904.0, |
| "loss": 1.85693359375, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.5546875, |
| "reward/margin": 0.2734375, |
| "reward/rejected": -1.828125, |
| "step": 449 |
| }, |
| { |
| "approx. KL/chosen": 4.0, |
| "approx. KL/rejected": 536.0, |
| "epoch": 0.4610655737704918, |
| "grad_norm": 45.132293701171875, |
| "learning_rate": 5.5149110531795656e-06, |
| "logp/chosen": -856.0, |
| "logp/rejected": -1280.0, |
| "loss": 0.74853515625, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.0, |
| "reward/margin": 5.0, |
| "reward/rejected": -5.0, |
| "step": 450 |
| }, |
| { |
| "approx. KL/chosen": 26.5, |
| "approx. KL/rejected": 564.0, |
| "epoch": 0.46209016393442626, |
| "grad_norm": 10.993429183959961, |
| "learning_rate": 5.502656001459011e-06, |
| "logp/chosen": -1232.0, |
| "logp/rejected": -1720.0, |
| "loss": 0.6953125, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.1005859375, |
| "reward/margin": 5.28125, |
| "reward/rejected": -5.375, |
| "step": 451 |
| }, |
| { |
| "approx. KL/chosen": 33.25, |
| "approx. KL/rejected": 133.0, |
| "epoch": 0.46311475409836067, |
| "grad_norm": 78.93635559082031, |
| "learning_rate": 5.490386832482274e-06, |
| "logp/chosen": -896.0, |
| "logp/rejected": -1072.0, |
| "loss": 1.346435546875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.050048828125, |
| "reward/margin": 1.828125, |
| "reward/rejected": -1.875, |
| "step": 452 |
| }, |
| { |
| "approx. KL/chosen": 73.5, |
| "approx. KL/rejected": 230.0, |
| "epoch": 0.4641393442622951, |
| "grad_norm": 109.22118377685547, |
| "learning_rate": 5.478103703330987e-06, |
| "logp/chosen": -656.0, |
| "logp/rejected": -884.0, |
| "loss": 1.7421875, |
| "nll_loss": 0.482421875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.6015625, |
| "reward/margin": 1.15625, |
| "reward/rejected": -2.75, |
| "step": 453 |
| }, |
| { |
| "approx. KL/chosen": 25.5, |
| "approx. KL/rejected": 76.5, |
| "epoch": 0.4651639344262295, |
| "grad_norm": 56.78947448730469, |
| "learning_rate": 5.465806771265508e-06, |
| "logp/chosen": -716.0, |
| "logp/rejected": -800.0, |
| "loss": 0.963134765625, |
| "nll_loss": 0.4921875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.7265625, |
| "reward/margin": 1.2265625, |
| "reward/rejected": -1.953125, |
| "step": 454 |
| }, |
| { |
| "approx. KL/chosen": 90.0, |
| "approx. KL/rejected": 167.0, |
| "epoch": 0.4661885245901639, |
| "grad_norm": 103.76181030273438, |
| "learning_rate": 5.4534961937229185e-06, |
| "logp/chosen": -812.0, |
| "logp/rejected": -852.0, |
| "loss": 1.833984375, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.6484375, |
| "reward/margin": 1.0, |
| "reward/rejected": -2.65625, |
| "step": 455 |
| }, |
| { |
| "approx. KL/chosen": 23.25, |
| "approx. KL/rejected": 282.0, |
| "epoch": 0.4672131147540984, |
| "grad_norm": 123.97257232666016, |
| "learning_rate": 5.441172128314999e-06, |
| "logp/chosen": -932.0, |
| "logp/rejected": -1736.0, |
| "loss": 1.5146484375, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.451171875, |
| "reward/margin": 2.90625, |
| "reward/rejected": -3.359375, |
| "step": 456 |
| }, |
| { |
| "approx. KL/chosen": 29.25, |
| "approx. KL/rejected": 143.0, |
| "epoch": 0.4682377049180328, |
| "grad_norm": 62.50522994995117, |
| "learning_rate": 5.428834732826217e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1424.0, |
| "loss": 1.2099609375, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 2.0, |
| "reward/rejected": -2.859375, |
| "step": 457 |
| }, |
| { |
| "approx. KL/chosen": 62.25, |
| "approx. KL/rejected": 312.0, |
| "epoch": 0.4692622950819672, |
| "grad_norm": 54.972686767578125, |
| "learning_rate": 5.416484165211701e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1328.0, |
| "loss": 0.9697265625, |
| "nll_loss": 0.671875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.3515625, |
| "reward/margin": 4.21875, |
| "reward/rejected": -3.875, |
| "step": 458 |
| }, |
| { |
| "approx. KL/chosen": 25.0, |
| "approx. KL/rejected": 386.0, |
| "epoch": 0.4702868852459016, |
| "grad_norm": 44.167415618896484, |
| "learning_rate": 5.404120583595226e-06, |
| "logp/chosen": -848.0, |
| "logp/rejected": -1168.0, |
| "loss": 0.859619140625, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.1748046875, |
| "reward/margin": 3.78125, |
| "reward/rejected": -3.953125, |
| "step": 459 |
| }, |
| { |
| "approx. KL/chosen": 91.5, |
| "approx. KL/rejected": 128.0, |
| "epoch": 0.4713114754098361, |
| "grad_norm": 125.11643981933594, |
| "learning_rate": 5.3917441462671815e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1136.0, |
| "loss": 1.666015625, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.9765625, |
| "reward/margin": 0.53515625, |
| "reward/rejected": -2.515625, |
| "step": 460 |
| }, |
| { |
| "approx. KL/chosen": 30.0, |
| "approx. KL/rejected": 348.0, |
| "epoch": 0.4723360655737705, |
| "grad_norm": 55.21257781982422, |
| "learning_rate": 5.379355011682548e-06, |
| "logp/chosen": -1072.0, |
| "logp/rejected": -1320.0, |
| "loss": 0.88037109375, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.400390625, |
| "reward/margin": 4.71875, |
| "reward/rejected": -4.3125, |
| "step": 461 |
| }, |
| { |
| "approx. KL/chosen": 48.75, |
| "approx. KL/rejected": 177.0, |
| "epoch": 0.4733606557377049, |
| "grad_norm": 71.10903930664062, |
| "learning_rate": 5.36695333845887e-06, |
| "logp/chosen": -792.0, |
| "logp/rejected": -1600.0, |
| "loss": 1.26416015625, |
| "nll_loss": 0.66796875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.4296875, |
| "reward/margin": 1.2734375, |
| "reward/rejected": -2.703125, |
| "step": 462 |
| }, |
| { |
| "approx. KL/chosen": 29.5, |
| "approx. KL/rejected": 222.0, |
| "epoch": 0.47438524590163933, |
| "grad_norm": 33.25880813598633, |
| "learning_rate": 5.3545392853742245e-06, |
| "logp/chosen": -1080.0, |
| "logp/rejected": -1704.0, |
| "loss": 0.76220703125, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.275390625, |
| "reward/margin": 3.671875, |
| "reward/rejected": -3.390625, |
| "step": 463 |
| }, |
| { |
| "approx. KL/chosen": 48.25, |
| "approx. KL/rejected": 192.0, |
| "epoch": 0.47540983606557374, |
| "grad_norm": 88.12850952148438, |
| "learning_rate": 5.342113011365185e-06, |
| "logp/chosen": -876.0, |
| "logp/rejected": -1056.0, |
| "loss": 1.44921875, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.15625, |
| "reward/margin": 1.7265625, |
| "reward/rejected": -2.875, |
| "step": 464 |
| }, |
| { |
| "approx. KL/chosen": 11.5625, |
| "approx. KL/rejected": 172.0, |
| "epoch": 0.4764344262295082, |
| "grad_norm": 48.956687927246094, |
| "learning_rate": 5.329674675524787e-06, |
| "logp/chosen": -496.0, |
| "logp/rejected": -544.0, |
| "loss": 0.848388671875, |
| "nll_loss": 0.419921875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.275390625, |
| "reward/margin": 2.59375, |
| "reward/rejected": -2.875, |
| "step": 465 |
| }, |
| { |
| "approx. KL/chosen": 23.75, |
| "approx. KL/rejected": 185.0, |
| "epoch": 0.4774590163934426, |
| "grad_norm": 50.290916442871094, |
| "learning_rate": 5.3172244371005014e-06, |
| "logp/chosen": -772.0, |
| "logp/rejected": -1184.0, |
| "loss": 0.814453125, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 2.796875, |
| "reward/rejected": -3.359375, |
| "step": 466 |
| }, |
| { |
| "approx. KL/chosen": 26.0, |
| "approx. KL/rejected": 55.0, |
| "epoch": 0.47848360655737704, |
| "grad_norm": 52.94970703125, |
| "learning_rate": 5.304762455492178e-06, |
| "logp/chosen": -1232.0, |
| "logp/rejected": -1288.0, |
| "loss": 0.9072265625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.0, |
| "reward/margin": 1.703125, |
| "reward/rejected": -1.703125, |
| "step": 467 |
| }, |
| { |
| "approx. KL/chosen": 59.0, |
| "approx. KL/rejected": 26.25, |
| "epoch": 0.47950819672131145, |
| "grad_norm": 75.71891021728516, |
| "learning_rate": 5.2922888902500215e-06, |
| "logp/chosen": -900.0, |
| "logp/rejected": -908.0, |
| "loss": 1.265625, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.298828125, |
| "reward/margin": 0.828125, |
| "reward/rejected": -1.125, |
| "step": 468 |
| }, |
| { |
| "approx. KL/chosen": 104.0, |
| "approx. KL/rejected": 69.0, |
| "epoch": 0.4805327868852459, |
| "grad_norm": 177.3874053955078, |
| "learning_rate": 5.2798039010725334e-06, |
| "logp/chosen": -1344.0, |
| "logp/rejected": -904.0, |
| "loss": 2.66357421875, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -2.359375, |
| "reward/margin": -1.203125, |
| "reward/rejected": -1.1484375, |
| "step": 469 |
| }, |
| { |
| "approx. KL/chosen": 11.9375, |
| "approx. KL/rejected": 133.0, |
| "epoch": 0.48155737704918034, |
| "grad_norm": 23.56594467163086, |
| "learning_rate": 5.267307647804483e-06, |
| "logp/chosen": -564.0, |
| "logp/rejected": -860.0, |
| "loss": 0.8740234375, |
| "nll_loss": 0.5, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.125, |
| "reward/margin": 2.203125, |
| "reward/rejected": -2.328125, |
| "step": 470 |
| }, |
| { |
| "approx. KL/chosen": 16.25, |
| "approx. KL/rejected": 104.5, |
| "epoch": 0.48258196721311475, |
| "grad_norm": 82.16121673583984, |
| "learning_rate": 5.254800290434849e-06, |
| "logp/chosen": -916.0, |
| "logp/rejected": -1528.0, |
| "loss": 0.922607421875, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.0498046875, |
| "reward/margin": 2.03125, |
| "reward/rejected": -2.078125, |
| "step": 471 |
| }, |
| { |
| "approx. KL/chosen": 66.0, |
| "approx. KL/rejected": 132.0, |
| "epoch": 0.48360655737704916, |
| "grad_norm": 86.06763458251953, |
| "learning_rate": 5.242281989094777e-06, |
| "logp/chosen": -1216.0, |
| "logp/rejected": -1520.0, |
| "loss": 1.20361328125, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 1.421875, |
| "reward/rejected": -2.21875, |
| "step": 472 |
| }, |
| { |
| "approx. KL/chosen": 15.75, |
| "approx. KL/rejected": 418.0, |
| "epoch": 0.48463114754098363, |
| "grad_norm": 30.829097747802734, |
| "learning_rate": 5.229752904055527e-06, |
| "logp/chosen": -592.0, |
| "logp/rejected": -1288.0, |
| "loss": 0.750244140625, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 4.15625, |
| "reward/rejected": -4.8125, |
| "step": 473 |
| }, |
| { |
| "approx. KL/chosen": 34.25, |
| "approx. KL/rejected": 41.0, |
| "epoch": 0.48565573770491804, |
| "grad_norm": 92.55126190185547, |
| "learning_rate": 5.217213195726421e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -1152.0, |
| "loss": 1.5615234375, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.75, |
| "reward/margin": 0.55078125, |
| "reward/rejected": -1.3046875, |
| "step": 474 |
| }, |
| { |
| "approx. KL/chosen": 102.0, |
| "approx. KL/rejected": 215.0, |
| "epoch": 0.48668032786885246, |
| "grad_norm": 120.78277587890625, |
| "learning_rate": 5.204663024652793e-06, |
| "logp/chosen": -1152.0, |
| "logp/rejected": -1400.0, |
| "loss": 1.9150390625, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.6015625, |
| "reward/margin": 0.9296875, |
| "reward/rejected": -2.53125, |
| "step": 475 |
| }, |
| { |
| "approx. KL/chosen": 53.25, |
| "approx. KL/rejected": 137.0, |
| "epoch": 0.48770491803278687, |
| "grad_norm": 124.65849304199219, |
| "learning_rate": 5.192102551513931e-06, |
| "logp/chosen": -932.0, |
| "logp/rejected": -1312.0, |
| "loss": 1.5966796875, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.349609375, |
| "reward/margin": 1.75, |
| "reward/rejected": -2.109375, |
| "step": 476 |
| }, |
| { |
| "approx. KL/chosen": 42.5, |
| "approx. KL/rejected": 400.0, |
| "epoch": 0.4887295081967213, |
| "grad_norm": 86.01321411132812, |
| "learning_rate": 5.179531937121018e-06, |
| "logp/chosen": -660.0, |
| "logp/rejected": -856.0, |
| "loss": 1.171875, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 2.984375, |
| "reward/rejected": -4.09375, |
| "step": 477 |
| }, |
| { |
| "approx. KL/chosen": 24.0, |
| "approx. KL/rejected": 268.0, |
| "epoch": 0.48975409836065575, |
| "grad_norm": 102.2696762084961, |
| "learning_rate": 5.166951342415076e-06, |
| "logp/chosen": -1064.0, |
| "logp/rejected": -1400.0, |
| "loss": 1.10888671875, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 3.203125, |
| "reward/rejected": -3.40625, |
| "step": 478 |
| }, |
| { |
| "approx. KL/chosen": 89.0, |
| "approx. KL/rejected": 198.0, |
| "epoch": 0.49077868852459017, |
| "grad_norm": 140.55044555664062, |
| "learning_rate": 5.154360928464907e-06, |
| "logp/chosen": -1472.0, |
| "logp/rejected": -1488.0, |
| "loss": 2.07373046875, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.3046875, |
| "reward/margin": 1.1015625, |
| "reward/rejected": -2.40625, |
| "step": 479 |
| }, |
| { |
| "approx. KL/chosen": 73.0, |
| "approx. KL/rejected": 242.0, |
| "epoch": 0.4918032786885246, |
| "grad_norm": 90.40853118896484, |
| "learning_rate": 5.141760856465021e-06, |
| "logp/chosen": -676.0, |
| "logp/rejected": -1096.0, |
| "loss": 2.06689453125, |
| "nll_loss": 0.64453125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.8515625, |
| "reward/margin": 1.2265625, |
| "reward/rejected": -3.078125, |
| "step": 480 |
| }, |
| { |
| "approx. KL/chosen": 43.5, |
| "approx. KL/rejected": 170.0, |
| "epoch": 0.492827868852459, |
| "grad_norm": 57.035865783691406, |
| "learning_rate": 5.129151287733587e-06, |
| "logp/chosen": -792.0, |
| "logp/rejected": -944.0, |
| "loss": 1.19482421875, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.3984375, |
| "reward/margin": 1.65625, |
| "reward/rejected": -3.0625, |
| "step": 481 |
| }, |
| { |
| "approx. KL/chosen": 34.5, |
| "approx. KL/rejected": 292.0, |
| "epoch": 0.49385245901639346, |
| "grad_norm": 42.327239990234375, |
| "learning_rate": 5.116532383710358e-06, |
| "logp/chosen": -904.0, |
| "logp/rejected": -1088.0, |
| "loss": 0.81201171875, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -1.0, |
| "reward/margin": 3.15625, |
| "reward/rejected": -4.15625, |
| "step": 482 |
| }, |
| { |
| "approx. KL/chosen": 42.25, |
| "approx. KL/rejected": 70.5, |
| "epoch": 0.4948770491803279, |
| "grad_norm": 141.2617645263672, |
| "learning_rate": 5.103904305954606e-06, |
| "logp/chosen": -1240.0, |
| "logp/rejected": -1320.0, |
| "loss": 1.5810546875, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.453125, |
| "reward/margin": 0.3203125, |
| "reward/rejected": -1.7734375, |
| "step": 483 |
| }, |
| { |
| "approx. KL/chosen": 42.0, |
| "approx. KL/rejected": 125.0, |
| "epoch": 0.4959016393442623, |
| "grad_norm": 101.61614990234375, |
| "learning_rate": 5.0912672161430515e-06, |
| "logp/chosen": -1012.0, |
| "logp/rejected": -1232.0, |
| "loss": 1.328125, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.0, |
| "reward/margin": 1.5, |
| "reward/rejected": -2.5, |
| "step": 484 |
| }, |
| { |
| "approx. KL/chosen": 43.0, |
| "approx. KL/rejected": 218.0, |
| "epoch": 0.4969262295081967, |
| "grad_norm": 94.55696105957031, |
| "learning_rate": 5.078621276067802e-06, |
| "logp/chosen": -1304.0, |
| "logp/rejected": -1800.0, |
| "loss": 1.01171875, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 2.75, |
| "reward/rejected": -3.65625, |
| "step": 485 |
| }, |
| { |
| "approx. KL/chosen": 23.25, |
| "approx. KL/rejected": 224.0, |
| "epoch": 0.4979508196721312, |
| "grad_norm": 72.43021392822266, |
| "learning_rate": 5.065966647634268e-06, |
| "logp/chosen": -904.0, |
| "logp/rejected": -1304.0, |
| "loss": 1.02587890625, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.25, |
| "reward/margin": 2.921875, |
| "reward/rejected": -3.171875, |
| "step": 486 |
| }, |
| { |
| "approx. KL/chosen": 18.25, |
| "approx. KL/rejected": 68.0, |
| "epoch": 0.4989754098360656, |
| "grad_norm": 31.581226348876953, |
| "learning_rate": 5.053303492859101e-06, |
| "logp/chosen": -660.0, |
| "logp/rejected": -920.0, |
| "loss": 0.7216796875, |
| "nll_loss": 0.52734375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.349609375, |
| "reward/margin": 2.3125, |
| "reward/rejected": -1.953125, |
| "step": 487 |
| }, |
| { |
| "approx. KL/chosen": 29.0, |
| "approx. KL/rejected": 132.0, |
| "epoch": 0.5, |
| "grad_norm": 26.385421752929688, |
| "learning_rate": 5.0406319738681125e-06, |
| "logp/chosen": -1128.0, |
| "logp/rejected": -1664.0, |
| "loss": 0.775390625, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.7265625, |
| "reward/margin": 3.375, |
| "reward/rejected": -2.65625, |
| "step": 488 |
| }, |
| { |
| "approx. KL/chosen": 42.75, |
| "approx. KL/rejected": 260.0, |
| "epoch": 0.5010245901639344, |
| "grad_norm": 84.4918441772461, |
| "learning_rate": 5.027952252894202e-06, |
| "logp/chosen": -652.0, |
| "logp/rejected": -876.0, |
| "loss": 1.1541748046875, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.75, |
| "reward/margin": 2.671875, |
| "reward/rejected": -3.421875, |
| "step": 489 |
| }, |
| { |
| "approx. KL/chosen": 34.0, |
| "approx. KL/rejected": 86.0, |
| "epoch": 0.5020491803278688, |
| "grad_norm": 98.33368682861328, |
| "learning_rate": 5.0152644922752785e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1272.0, |
| "loss": 1.1376953125, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 1.53125, |
| "reward/rejected": -1.9296875, |
| "step": 490 |
| }, |
| { |
| "approx. KL/chosen": 65.0, |
| "approx. KL/rejected": 171.0, |
| "epoch": 0.5030737704918032, |
| "grad_norm": 56.90475082397461, |
| "learning_rate": 5.00256885445218e-06, |
| "logp/chosen": -780.0, |
| "logp/rejected": -1128.0, |
| "loss": 0.9248046875, |
| "nll_loss": 0.515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.2734375, |
| "reward/margin": 1.4765625, |
| "reward/rejected": -2.75, |
| "step": 491 |
| }, |
| { |
| "approx. KL/chosen": 39.5, |
| "approx. KL/rejected": 64.0, |
| "epoch": 0.5040983606557377, |
| "grad_norm": 76.93305206298828, |
| "learning_rate": 4.989865501966601e-06, |
| "logp/chosen": -1224.0, |
| "logp/rejected": -1000.0, |
| "loss": 1.185546875, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.32421875, |
| "reward/margin": 1.6015625, |
| "reward/rejected": -1.9296875, |
| "step": 492 |
| }, |
| { |
| "approx. KL/chosen": 16.0, |
| "approx. KL/rejected": 194.0, |
| "epoch": 0.5051229508196722, |
| "grad_norm": 108.85711669921875, |
| "learning_rate": 4.977154597459002e-06, |
| "logp/chosen": -1152.0, |
| "logp/rejected": -1488.0, |
| "loss": 1.283203125, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 1.796875, |
| "reward/rejected": -2.1875, |
| "step": 493 |
| }, |
| { |
| "approx. KL/chosen": 30.0, |
| "approx. KL/rejected": 165.0, |
| "epoch": 0.5061475409836066, |
| "grad_norm": 70.26461791992188, |
| "learning_rate": 4.9644363036665314e-06, |
| "logp/chosen": -980.0, |
| "logp/rejected": -1528.0, |
| "loss": 1.18359375, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0, |
| "reward/margin": 1.8515625, |
| "reward/rejected": -2.859375, |
| "step": 494 |
| }, |
| { |
| "approx. KL/chosen": 37.5, |
| "approx. KL/rejected": 198.0, |
| "epoch": 0.507172131147541, |
| "grad_norm": 46.402549743652344, |
| "learning_rate": 4.951710783420947e-06, |
| "logp/chosen": -772.0, |
| "logp/rejected": -1160.0, |
| "loss": 0.76025390625, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.2001953125, |
| "reward/margin": 3.15625, |
| "reward/rejected": -2.953125, |
| "step": 495 |
| }, |
| { |
| "approx. KL/chosen": 34.25, |
| "approx. KL/rejected": 72.0, |
| "epoch": 0.5081967213114754, |
| "grad_norm": 43.83592224121094, |
| "learning_rate": 4.9389781996465215e-06, |
| "logp/chosen": -672.0, |
| "logp/rejected": -1016.0, |
| "loss": 1.0419921875, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.92578125, |
| "reward/margin": 1.1953125, |
| "reward/rejected": -2.125, |
| "step": 496 |
| }, |
| { |
| "approx. KL/chosen": 80.5, |
| "approx. KL/rejected": 81.5, |
| "epoch": 0.5092213114754098, |
| "grad_norm": 124.76813507080078, |
| "learning_rate": 4.926238715357964e-06, |
| "logp/chosen": -1240.0, |
| "logp/rejected": -984.0, |
| "loss": 1.74658203125, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -2.078125, |
| "reward/margin": -0.0732421875, |
| "reward/rejected": -2.0, |
| "step": 497 |
| }, |
| { |
| "approx. KL/chosen": 23.0, |
| "approx. KL/rejected": 162.0, |
| "epoch": 0.5102459016393442, |
| "grad_norm": 63.7852668762207, |
| "learning_rate": 4.913492493658331e-06, |
| "logp/chosen": -592.0, |
| "logp/rejected": -748.0, |
| "loss": 1.49951171875, |
| "nll_loss": 0.703125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.375, |
| "reward/margin": 2.109375, |
| "reward/rejected": -2.484375, |
| "step": 498 |
| }, |
| { |
| "approx. KL/chosen": 22.5, |
| "approx. KL/rejected": 302.0, |
| "epoch": 0.5112704918032787, |
| "grad_norm": 27.282236099243164, |
| "learning_rate": 4.900739697736937e-06, |
| "logp/chosen": -864.0, |
| "logp/rejected": -1104.0, |
| "loss": 0.701171875, |
| "nll_loss": 0.48046875, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.7890625, |
| "reward/margin": 2.84375, |
| "reward/rejected": -3.625, |
| "step": 499 |
| }, |
| { |
| "approx. KL/chosen": 71.0, |
| "approx. KL/rejected": 156.0, |
| "epoch": 0.5122950819672131, |
| "grad_norm": 88.5564956665039, |
| "learning_rate": 4.887980490867267e-06, |
| "logp/chosen": -976.0, |
| "logp/rejected": -1208.0, |
| "loss": 1.0546875, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.5, |
| "reward/margin": 2.359375, |
| "reward/rejected": -2.859375, |
| "step": 500 |
| }, |
| { |
| "approx. KL/chosen": 57.25, |
| "approx. KL/rejected": 368.0, |
| "epoch": 0.5133196721311475, |
| "grad_norm": 108.88277435302734, |
| "learning_rate": 4.87521503640488e-06, |
| "logp/chosen": -1248.0, |
| "logp/rejected": -1464.0, |
| "loss": 1.552978515625, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 2.609375, |
| "reward/rejected": -3.15625, |
| "step": 501 |
| }, |
| { |
| "approx. KL/chosen": 62.25, |
| "approx. KL/rejected": 296.0, |
| "epoch": 0.514344262295082, |
| "grad_norm": 81.80404663085938, |
| "learning_rate": 4.862443497785332e-06, |
| "logp/chosen": -1088.0, |
| "logp/rejected": -1296.0, |
| "loss": 0.92578125, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 3.546875, |
| "reward/rejected": -4.09375, |
| "step": 502 |
| }, |
| { |
| "approx. KL/chosen": 40.25, |
| "approx. KL/rejected": 230.0, |
| "epoch": 0.5153688524590164, |
| "grad_norm": 67.22120666503906, |
| "learning_rate": 4.849666038522065e-06, |
| "logp/chosen": -696.0, |
| "logp/rejected": -772.0, |
| "loss": 1.14208984375, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.078125, |
| "reward/margin": 2.359375, |
| "reward/rejected": -3.421875, |
| "step": 503 |
| }, |
| { |
| "approx. KL/chosen": 43.5, |
| "approx. KL/rejected": 195.0, |
| "epoch": 0.5163934426229508, |
| "grad_norm": 53.11055374145508, |
| "learning_rate": 4.836882822204329e-06, |
| "logp/chosen": -776.0, |
| "logp/rejected": -1048.0, |
| "loss": 0.89697265625, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.40234375, |
| "reward/margin": 3.203125, |
| "reward/rejected": -2.796875, |
| "step": 504 |
| }, |
| { |
| "approx. KL/chosen": 39.75, |
| "approx. KL/rejected": 253.0, |
| "epoch": 0.5174180327868853, |
| "grad_norm": 90.59297180175781, |
| "learning_rate": 4.824094012495078e-06, |
| "logp/chosen": -1168.0, |
| "logp/rejected": -1192.0, |
| "loss": 1.4248046875, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.2734375, |
| "reward/margin": 1.78125, |
| "reward/rejected": -3.0625, |
| "step": 505 |
| }, |
| { |
| "approx. KL/chosen": 115.0, |
| "approx. KL/rejected": 196.0, |
| "epoch": 0.5184426229508197, |
| "grad_norm": 96.98806762695312, |
| "learning_rate": 4.8112997731288785e-06, |
| "logp/chosen": -1416.0, |
| "logp/rejected": -1440.0, |
| "loss": 1.01953125, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.703125, |
| "reward/margin": 1.5546875, |
| "reward/rejected": -3.25, |
| "step": 506 |
| }, |
| { |
| "approx. KL/chosen": 26.25, |
| "approx. KL/rejected": 202.0, |
| "epoch": 0.5194672131147541, |
| "grad_norm": 56.949859619140625, |
| "learning_rate": 4.798500267909816e-06, |
| "logp/chosen": -900.0, |
| "logp/rejected": -1104.0, |
| "loss": 0.8466796875, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.953125, |
| "reward/margin": 2.359375, |
| "reward/rejected": -3.296875, |
| "step": 507 |
| }, |
| { |
| "approx. KL/chosen": 33.75, |
| "approx. KL/rejected": 98.5, |
| "epoch": 0.5204918032786885, |
| "grad_norm": 58.28824234008789, |
| "learning_rate": 4.785695660709388e-06, |
| "logp/chosen": -552.0, |
| "logp/rejected": -688.0, |
| "loss": 1.2607421875, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.265625, |
| "reward/margin": 1.0390625, |
| "reward/rejected": -2.296875, |
| "step": 508 |
| }, |
| { |
| "approx. KL/chosen": 47.25, |
| "approx. KL/rejected": 153.0, |
| "epoch": 0.5215163934426229, |
| "grad_norm": 56.84836196899414, |
| "learning_rate": 4.772886115464419e-06, |
| "logp/chosen": -1024.0, |
| "logp/rejected": -1328.0, |
| "loss": 0.82275390625, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.0546875, |
| "reward/margin": 2.046875, |
| "reward/rejected": -3.109375, |
| "step": 509 |
| }, |
| { |
| "approx. KL/chosen": 66.5, |
| "approx. KL/rejected": 241.0, |
| "epoch": 0.5225409836065574, |
| "grad_norm": 100.69557189941406, |
| "learning_rate": 4.76007179617495e-06, |
| "logp/chosen": -916.0, |
| "logp/rejected": -1256.0, |
| "loss": 1.68994140625, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0859375, |
| "reward/margin": 1.7734375, |
| "reward/rejected": -2.859375, |
| "step": 510 |
| }, |
| { |
| "approx. KL/chosen": 30.25, |
| "approx. KL/rejected": 380.0, |
| "epoch": 0.5235655737704918, |
| "grad_norm": 14.748980522155762, |
| "learning_rate": 4.747252866902146e-06, |
| "logp/chosen": -892.0, |
| "logp/rejected": -1272.0, |
| "loss": 0.58984375, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.1494140625, |
| "reward/margin": 4.875, |
| "reward/rejected": -4.71875, |
| "step": 511 |
| }, |
| { |
| "approx. KL/chosen": 40.25, |
| "approx. KL/rejected": 218.0, |
| "epoch": 0.5245901639344263, |
| "grad_norm": 105.5616226196289, |
| "learning_rate": 4.734429491766194e-06, |
| "logp/chosen": -1144.0, |
| "logp/rejected": -1368.0, |
| "loss": 1.646728515625, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.1484375, |
| "reward/margin": 1.3828125, |
| "reward/rejected": -2.53125, |
| "step": 512 |
| }, |
| { |
| "approx. KL/chosen": 16.5, |
| "approx. KL/rejected": 180.0, |
| "epoch": 0.5256147540983607, |
| "grad_norm": 33.64626693725586, |
| "learning_rate": 4.721601834944202e-06, |
| "logp/chosen": -696.0, |
| "logp/rejected": -1020.0, |
| "loss": 0.81494140625, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 2.625, |
| "reward/rejected": -3.03125, |
| "step": 513 |
| }, |
| { |
| "approx. KL/chosen": 18.25, |
| "approx. KL/rejected": 82.0, |
| "epoch": 0.5266393442622951, |
| "grad_norm": 46.55826950073242, |
| "learning_rate": 4.708770060668091e-06, |
| "logp/chosen": -940.0, |
| "logp/rejected": -868.0, |
| "loss": 0.8779296875, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.050048828125, |
| "reward/margin": 1.8828125, |
| "reward/rejected": -1.8359375, |
| "step": 514 |
| }, |
| { |
| "approx. KL/chosen": 33.5, |
| "approx. KL/rejected": 179.0, |
| "epoch": 0.5276639344262295, |
| "grad_norm": 96.2533187866211, |
| "learning_rate": 4.695934333222506e-06, |
| "logp/chosen": -884.0, |
| "logp/rejected": -1040.0, |
| "loss": 1.609375, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.201171875, |
| "reward/margin": 2.296875, |
| "reward/rejected": -2.109375, |
| "step": 515 |
| }, |
| { |
| "approx. KL/chosen": 40.5, |
| "approx. KL/rejected": 102.0, |
| "epoch": 0.5286885245901639, |
| "grad_norm": 95.06312561035156, |
| "learning_rate": 4.683094816942698e-06, |
| "logp/chosen": -1096.0, |
| "logp/rejected": -1512.0, |
| "loss": 1.0771484375, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 1.359375, |
| "reward/rejected": -2.25, |
| "step": 516 |
| }, |
| { |
| "approx. KL/chosen": 36.5, |
| "approx. KL/rejected": 316.0, |
| "epoch": 0.5297131147540983, |
| "grad_norm": 46.02762222290039, |
| "learning_rate": 4.670251676212427e-06, |
| "logp/chosen": -748.0, |
| "logp/rejected": -1160.0, |
| "loss": 1.205322265625, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.150390625, |
| "reward/margin": 2.890625, |
| "reward/rejected": -2.734375, |
| "step": 517 |
| }, |
| { |
| "approx. KL/chosen": 20.25, |
| "approx. KL/rejected": 372.0, |
| "epoch": 0.5307377049180327, |
| "grad_norm": 81.28799438476562, |
| "learning_rate": 4.657405075461859e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1472.0, |
| "loss": 1.2216796875, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 3.09375, |
| "reward/rejected": -3.953125, |
| "step": 518 |
| }, |
| { |
| "approx. KL/chosen": 22.25, |
| "approx. KL/rejected": 204.0, |
| "epoch": 0.5317622950819673, |
| "grad_norm": 45.05597686767578, |
| "learning_rate": 4.644555179165455e-06, |
| "logp/chosen": -932.0, |
| "logp/rejected": -1024.0, |
| "loss": 0.90576171875, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 2.625, |
| "reward/rejected": -3.28125, |
| "step": 519 |
| }, |
| { |
| "approx. KL/chosen": 63.0, |
| "approx. KL/rejected": 172.0, |
| "epoch": 0.5327868852459017, |
| "grad_norm": 85.08966064453125, |
| "learning_rate": 4.631702151839874e-06, |
| "logp/chosen": -1064.0, |
| "logp/rejected": -1312.0, |
| "loss": 1.3505859375, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.703125, |
| "reward/margin": 2.484375, |
| "reward/rejected": -3.171875, |
| "step": 520 |
| }, |
| { |
| "approx. KL/chosen": 49.25, |
| "approx. KL/rejected": 186.0, |
| "epoch": 0.5338114754098361, |
| "grad_norm": 115.0302963256836, |
| "learning_rate": 4.618846158041856e-06, |
| "logp/chosen": -1224.0, |
| "logp/rejected": -1136.0, |
| "loss": 1.3486328125, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.1484375, |
| "reward/margin": 1.640625, |
| "reward/rejected": -2.78125, |
| "step": 521 |
| }, |
| { |
| "approx. KL/chosen": 29.75, |
| "approx. KL/rejected": 154.0, |
| "epoch": 0.5348360655737705, |
| "grad_norm": 54.607967376708984, |
| "learning_rate": 4.6059873623661225e-06, |
| "logp/chosen": -712.0, |
| "logp/rejected": -1136.0, |
| "loss": 1.04345703125, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 1.953125, |
| "reward/rejected": -2.796875, |
| "step": 522 |
| }, |
| { |
| "approx. KL/chosen": 22.0, |
| "approx. KL/rejected": 308.0, |
| "epoch": 0.5358606557377049, |
| "grad_norm": 32.37906265258789, |
| "learning_rate": 4.593125929443269e-06, |
| "logp/chosen": -1012.0, |
| "logp/rejected": -1712.0, |
| "loss": 0.7744140625, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.400390625, |
| "reward/margin": 4.84375, |
| "reward/rejected": -4.46875, |
| "step": 523 |
| }, |
| { |
| "approx. KL/chosen": 18.0, |
| "approx. KL/rejected": 147.0, |
| "epoch": 0.5368852459016393, |
| "grad_norm": 64.5525894165039, |
| "learning_rate": 4.580262023937654e-06, |
| "logp/chosen": -816.0, |
| "logp/rejected": -872.0, |
| "loss": 1.173828125, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 2.296875, |
| "reward/rejected": -2.703125, |
| "step": 524 |
| }, |
| { |
| "approx. KL/chosen": 42.25, |
| "approx. KL/rejected": 174.0, |
| "epoch": 0.5379098360655737, |
| "grad_norm": 88.68099212646484, |
| "learning_rate": 4.567395810545292e-06, |
| "logp/chosen": -988.0, |
| "logp/rejected": -1416.0, |
| "loss": 1.67919921875, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 1.7265625, |
| "reward/rejected": -2.578125, |
| "step": 525 |
| }, |
| { |
| "approx. KL/chosen": 59.25, |
| "approx. KL/rejected": 350.0, |
| "epoch": 0.5389344262295082, |
| "grad_norm": 112.77783203125, |
| "learning_rate": 4.554527453991747e-06, |
| "logp/chosen": -944.0, |
| "logp/rejected": -1584.0, |
| "loss": 1.939453125, |
| "nll_loss": 0.443359375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.2890625, |
| "reward/margin": 1.9140625, |
| "reward/rejected": -3.203125, |
| "step": 526 |
| }, |
| { |
| "approx. KL/chosen": 10.25, |
| "approx. KL/rejected": 288.0, |
| "epoch": 0.5399590163934426, |
| "grad_norm": 19.78618621826172, |
| "learning_rate": 4.54165711903002e-06, |
| "logp/chosen": -996.0, |
| "logp/rejected": -1432.0, |
| "loss": 0.71875, |
| "nll_loss": 0.640625, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 3.953125, |
| "reward/rejected": -4.09375, |
| "step": 527 |
| }, |
| { |
| "approx. KL/chosen": 45.0, |
| "approx. KL/rejected": 92.0, |
| "epoch": 0.5409836065573771, |
| "grad_norm": 118.24066925048828, |
| "learning_rate": 4.528784970438443e-06, |
| "logp/chosen": -912.0, |
| "logp/rejected": -1120.0, |
| "loss": 1.5712890625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 0.7890625, |
| "reward/rejected": -1.6875, |
| "step": 528 |
| }, |
| { |
| "approx. KL/chosen": 170.0, |
| "approx. KL/rejected": 256.0, |
| "epoch": 0.5420081967213115, |
| "grad_norm": 81.81969451904297, |
| "learning_rate": 4.515911173018566e-06, |
| "logp/chosen": -748.0, |
| "logp/rejected": -696.0, |
| "loss": 2.14697265625, |
| "nll_loss": 0.52734375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.9140625, |
| "reward/margin": 2.09375, |
| "reward/rejected": -3.015625, |
| "step": 529 |
| }, |
| { |
| "approx. KL/chosen": 122.5, |
| "approx. KL/rejected": 356.0, |
| "epoch": 0.5430327868852459, |
| "grad_norm": 68.06129455566406, |
| "learning_rate": 4.503035891593051e-06, |
| "logp/chosen": -984.0, |
| "logp/rejected": -1176.0, |
| "loss": 0.946044921875, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -2.125, |
| "reward/margin": 2.03125, |
| "reward/rejected": -4.15625, |
| "step": 530 |
| }, |
| { |
| "approx. KL/chosen": 32.5, |
| "approx. KL/rejected": 378.0, |
| "epoch": 0.5440573770491803, |
| "grad_norm": 64.30060577392578, |
| "learning_rate": 4.490159291003556e-06, |
| "logp/chosen": -800.0, |
| "logp/rejected": -1080.0, |
| "loss": 1.17626953125, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 3.046875, |
| "reward/rejected": -3.359375, |
| "step": 531 |
| }, |
| { |
| "approx. KL/chosen": 65.0, |
| "approx. KL/rejected": 169.0, |
| "epoch": 0.5450819672131147, |
| "grad_norm": 93.92388916015625, |
| "learning_rate": 4.477281536108634e-06, |
| "logp/chosen": -692.0, |
| "logp/rejected": -1012.0, |
| "loss": 2.15673828125, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 0.625, |
| "reward/rejected": -1.53125, |
| "step": 532 |
| }, |
| { |
| "approx. KL/chosen": 27.0, |
| "approx. KL/rejected": 278.0, |
| "epoch": 0.5461065573770492, |
| "grad_norm": 77.77955627441406, |
| "learning_rate": 4.464402791781611e-06, |
| "logp/chosen": -896.0, |
| "logp/rejected": -1168.0, |
| "loss": 1.253662109375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 2.703125, |
| "reward/rejected": -3.609375, |
| "step": 533 |
| }, |
| { |
| "approx. KL/chosen": 32.5, |
| "approx. KL/rejected": 170.0, |
| "epoch": 0.5471311475409836, |
| "grad_norm": 84.44503784179688, |
| "learning_rate": 4.4515232229084854e-06, |
| "logp/chosen": -1088.0, |
| "logp/rejected": -1168.0, |
| "loss": 1.24951171875, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 1.6171875, |
| "reward/rejected": -2.71875, |
| "step": 534 |
| }, |
| { |
| "approx. KL/chosen": 19.25, |
| "approx. KL/rejected": 242.0, |
| "epoch": 0.548155737704918, |
| "grad_norm": 27.846712112426758, |
| "learning_rate": 4.438642994385806e-06, |
| "logp/chosen": -904.0, |
| "logp/rejected": -1336.0, |
| "loss": 0.68310546875, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.25, |
| "reward/margin": 4.15625, |
| "reward/rejected": -3.90625, |
| "step": 535 |
| }, |
| { |
| "approx. KL/chosen": 13.5, |
| "approx. KL/rejected": 229.0, |
| "epoch": 0.5491803278688525, |
| "grad_norm": 55.234676361083984, |
| "learning_rate": 4.4257622711185745e-06, |
| "logp/chosen": -788.0, |
| "logp/rejected": -1120.0, |
| "loss": 0.9677734375, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 2.578125, |
| "reward/rejected": -2.875, |
| "step": 536 |
| }, |
| { |
| "approx. KL/chosen": 30.75, |
| "approx. KL/rejected": 356.0, |
| "epoch": 0.5502049180327869, |
| "grad_norm": 80.68743133544922, |
| "learning_rate": 4.4128812180181206e-06, |
| "logp/chosen": -856.0, |
| "logp/rejected": -1352.0, |
| "loss": 1.148681640625, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.75, |
| "reward/margin": 3.34375, |
| "reward/rejected": -4.09375, |
| "step": 537 |
| }, |
| { |
| "approx. KL/chosen": 37.25, |
| "approx. KL/rejected": 250.0, |
| "epoch": 0.5512295081967213, |
| "grad_norm": 57.607460021972656, |
| "learning_rate": 4.4e-06, |
| "logp/chosen": -976.0, |
| "logp/rejected": -1256.0, |
| "loss": 0.93408203125, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 3.15625, |
| "reward/rejected": -3.703125, |
| "step": 538 |
| }, |
| { |
| "approx. KL/chosen": 14.3125, |
| "approx. KL/rejected": 130.0, |
| "epoch": 0.5522540983606558, |
| "grad_norm": 42.9509162902832, |
| "learning_rate": 4.387118781981879e-06, |
| "logp/chosen": -712.0, |
| "logp/rejected": -868.0, |
| "loss": 0.96875, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.67578125, |
| "reward/margin": 1.75, |
| "reward/rejected": -2.421875, |
| "step": 539 |
| }, |
| { |
| "approx. KL/chosen": 169.0, |
| "approx. KL/rejected": 224.0, |
| "epoch": 0.5532786885245902, |
| "grad_norm": 142.73141479492188, |
| "learning_rate": 4.374237728881426e-06, |
| "logp/chosen": -1520.0, |
| "logp/rejected": -1224.0, |
| "loss": 2.603515625, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -2.109375, |
| "reward/margin": 0.24609375, |
| "reward/rejected": -2.359375, |
| "step": 540 |
| }, |
| { |
| "approx. KL/chosen": 86.0, |
| "approx. KL/rejected": 388.0, |
| "epoch": 0.5543032786885246, |
| "grad_norm": 120.24505615234375, |
| "learning_rate": 4.361357005614193e-06, |
| "logp/chosen": -1004.0, |
| "logp/rejected": -1024.0, |
| "loss": 1.98046875, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.5546875, |
| "reward/margin": 1.546875, |
| "reward/rejected": -3.09375, |
| "step": 541 |
| }, |
| { |
| "approx. KL/chosen": 58.25, |
| "approx. KL/rejected": 120.0, |
| "epoch": 0.555327868852459, |
| "grad_norm": 109.25729370117188, |
| "learning_rate": 4.348476777091515e-06, |
| "logp/chosen": -1008.0, |
| "logp/rejected": -1248.0, |
| "loss": 1.268798828125, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.1484375, |
| "reward/margin": 1.25, |
| "reward/rejected": -2.40625, |
| "step": 542 |
| }, |
| { |
| "approx. KL/chosen": 21.25, |
| "approx. KL/rejected": 432.0, |
| "epoch": 0.5563524590163934, |
| "grad_norm": 27.298694610595703, |
| "learning_rate": 4.335597208218388e-06, |
| "logp/chosen": -828.0, |
| "logp/rejected": -1200.0, |
| "loss": 0.6572265625, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.0255126953125, |
| "reward/margin": 5.375, |
| "reward/rejected": -5.375, |
| "step": 543 |
| }, |
| { |
| "approx. KL/chosen": 65.0, |
| "approx. KL/rejected": 151.0, |
| "epoch": 0.5573770491803278, |
| "grad_norm": 80.5621566772461, |
| "learning_rate": 4.322718463891366e-06, |
| "logp/chosen": -908.0, |
| "logp/rejected": -1288.0, |
| "loss": 1.44970703125, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.328125, |
| "reward/margin": 1.5546875, |
| "reward/rejected": -2.875, |
| "step": 544 |
| }, |
| { |
| "approx. KL/chosen": 34.25, |
| "approx. KL/rejected": 154.0, |
| "epoch": 0.5584016393442623, |
| "grad_norm": 79.11225891113281, |
| "learning_rate": 4.309840708996443e-06, |
| "logp/chosen": -716.0, |
| "logp/rejected": -1048.0, |
| "loss": 1.166015625, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 1.828125, |
| "reward/rejected": -2.46875, |
| "step": 545 |
| }, |
| { |
| "approx. KL/chosen": 46.0, |
| "approx. KL/rejected": 138.0, |
| "epoch": 0.5594262295081968, |
| "grad_norm": 48.62302017211914, |
| "learning_rate": 4.296964108406949e-06, |
| "logp/chosen": -912.0, |
| "logp/rejected": -1144.0, |
| "loss": 0.95703125, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 2.28125, |
| "reward/rejected": -2.671875, |
| "step": 546 |
| }, |
| { |
| "approx. KL/chosen": 25.0, |
| "approx. KL/rejected": 652.0, |
| "epoch": 0.5604508196721312, |
| "grad_norm": 34.53141784667969, |
| "learning_rate": 4.284088826981433e-06, |
| "logp/chosen": -1160.0, |
| "logp/rejected": -2400.0, |
| "loss": 0.87158203125, |
| "nll_loss": 0.65625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.5, |
| "reward/margin": 5.21875, |
| "reward/rejected": -5.71875, |
| "step": 547 |
| }, |
| { |
| "approx. KL/chosen": 25.25, |
| "approx. KL/rejected": 140.0, |
| "epoch": 0.5614754098360656, |
| "grad_norm": 55.77986145019531, |
| "learning_rate": 4.271215029561557e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -964.0, |
| "loss": 0.896484375, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.375, |
| "reward/margin": 2.375, |
| "reward/rejected": -2.75, |
| "step": 548 |
| }, |
| { |
| "approx. KL/chosen": 28.0, |
| "approx. KL/rejected": 55.75, |
| "epoch": 0.5625, |
| "grad_norm": 45.68193054199219, |
| "learning_rate": 4.258342880969979e-06, |
| "logp/chosen": -652.0, |
| "logp/rejected": -540.0, |
| "loss": 0.9248046875, |
| "nll_loss": 0.66015625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 1.7265625, |
| "reward/rejected": -1.9296875, |
| "step": 549 |
| }, |
| { |
| "approx. KL/chosen": 24.75, |
| "approx. KL/rejected": 98.5, |
| "epoch": 0.5635245901639344, |
| "grad_norm": 75.38268280029297, |
| "learning_rate": 4.245472546008253e-06, |
| "logp/chosen": -792.0, |
| "logp/rejected": -1032.0, |
| "loss": 1.064453125, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 1.3515625, |
| "reward/rejected": -2.0, |
| "step": 550 |
| }, |
| { |
| "approx. KL/chosen": 77.0, |
| "approx. KL/rejected": 145.0, |
| "epoch": 0.5645491803278688, |
| "grad_norm": 114.9805679321289, |
| "learning_rate": 4.232604189454707e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1368.0, |
| "loss": 1.546630859375, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.75, |
| "reward/margin": 0.75, |
| "reward/rejected": -2.5, |
| "step": 551 |
| }, |
| { |
| "approx. KL/chosen": 25.0, |
| "approx. KL/rejected": 254.0, |
| "epoch": 0.5655737704918032, |
| "grad_norm": 95.50062561035156, |
| "learning_rate": 4.219737976062347e-06, |
| "logp/chosen": -1248.0, |
| "logp/rejected": -1248.0, |
| "loss": 1.001953125, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.69921875, |
| "reward/margin": 3.734375, |
| "reward/rejected": -3.03125, |
| "step": 552 |
| }, |
| { |
| "approx. KL/chosen": 151.0, |
| "approx. KL/rejected": 312.0, |
| "epoch": 0.5665983606557377, |
| "grad_norm": 132.87538146972656, |
| "learning_rate": 4.206874070556731e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -1016.0, |
| "loss": 2.783203125, |
| "nll_loss": 0.640625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -2.453125, |
| "reward/margin": 0.2734375, |
| "reward/rejected": -2.71875, |
| "step": 553 |
| }, |
| { |
| "approx. KL/chosen": 90.5, |
| "approx. KL/rejected": 137.0, |
| "epoch": 0.5676229508196722, |
| "grad_norm": 135.89352416992188, |
| "learning_rate": 4.194012637633879e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1012.0, |
| "loss": 1.571533203125, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.3046875, |
| "reward/margin": 1.3984375, |
| "reward/rejected": -2.703125, |
| "step": 554 |
| }, |
| { |
| "approx. KL/chosen": 8.0625, |
| "approx. KL/rejected": 184.0, |
| "epoch": 0.5686475409836066, |
| "grad_norm": 55.54265213012695, |
| "learning_rate": 4.181153841958145e-06, |
| "logp/chosen": -816.0, |
| "logp/rejected": -1056.0, |
| "loss": 0.97412109375, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.0250244140625, |
| "reward/margin": 2.703125, |
| "reward/rejected": -2.734375, |
| "step": 555 |
| }, |
| { |
| "approx. KL/chosen": 25.25, |
| "approx. KL/rejected": 114.5, |
| "epoch": 0.569672131147541, |
| "grad_norm": 35.39381790161133, |
| "learning_rate": 4.168297848160127e-06, |
| "logp/chosen": -784.0, |
| "logp/rejected": -988.0, |
| "loss": 0.984375, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.17578125, |
| "reward/margin": 2.546875, |
| "reward/rejected": -2.359375, |
| "step": 556 |
| }, |
| { |
| "approx. KL/chosen": 165.0, |
| "approx. KL/rejected": 160.0, |
| "epoch": 0.5706967213114754, |
| "grad_norm": 94.35969543457031, |
| "learning_rate": 4.155444820834544e-06, |
| "logp/chosen": -1088.0, |
| "logp/rejected": -1448.0, |
| "loss": 1.9853515625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.71484375, |
| "reward/margin": 2.09375, |
| "reward/rejected": -2.8125, |
| "step": 557 |
| }, |
| { |
| "approx. KL/chosen": 27.125, |
| "approx. KL/rejected": 232.0, |
| "epoch": 0.5717213114754098, |
| "grad_norm": 41.62066650390625, |
| "learning_rate": 4.142594924538143e-06, |
| "logp/chosen": -776.0, |
| "logp/rejected": -736.0, |
| "loss": 0.71435546875, |
| "nll_loss": 0.455078125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.1689453125, |
| "reward/margin": 3.078125, |
| "reward/rejected": -3.25, |
| "step": 558 |
| }, |
| { |
| "approx. KL/chosen": 30.25, |
| "approx. KL/rejected": 728.0, |
| "epoch": 0.5727459016393442, |
| "grad_norm": 23.720674514770508, |
| "learning_rate": 4.129748323787573e-06, |
| "logp/chosen": -1128.0, |
| "logp/rejected": -1688.0, |
| "loss": 0.77783203125, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.44921875, |
| "reward/margin": 6.0, |
| "reward/rejected": -6.4375, |
| "step": 559 |
| }, |
| { |
| "approx. KL/chosen": 30.5, |
| "approx. KL/rejected": 190.0, |
| "epoch": 0.5737704918032787, |
| "grad_norm": 82.38387298583984, |
| "learning_rate": 4.116905183057303e-06, |
| "logp/chosen": -1056.0, |
| "logp/rejected": -1120.0, |
| "loss": 1.12255859375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.326171875, |
| "reward/margin": 2.578125, |
| "reward/rejected": -2.25, |
| "step": 560 |
| }, |
| { |
| "approx. KL/chosen": 109.0, |
| "approx. KL/rejected": 231.0, |
| "epoch": 0.5747950819672131, |
| "grad_norm": 98.65898895263672, |
| "learning_rate": 4.104065666777493e-06, |
| "logp/chosen": -1096.0, |
| "logp/rejected": -1632.0, |
| "loss": 1.55859375, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.04736328125, |
| "reward/margin": 2.796875, |
| "reward/rejected": -2.859375, |
| "step": 561 |
| }, |
| { |
| "approx. KL/chosen": 97.0, |
| "approx. KL/rejected": 350.0, |
| "epoch": 0.5758196721311475, |
| "grad_norm": 170.73411560058594, |
| "learning_rate": 4.091229939331909e-06, |
| "logp/chosen": -1448.0, |
| "logp/rejected": -1832.0, |
| "loss": 3.0029296875, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.5, |
| "reward/margin": 1.1015625, |
| "reward/rejected": -2.609375, |
| "step": 562 |
| }, |
| { |
| "approx. KL/chosen": 32.25, |
| "approx. KL/rejected": 105.0, |
| "epoch": 0.576844262295082, |
| "grad_norm": 58.04389953613281, |
| "learning_rate": 4.078398165055799e-06, |
| "logp/chosen": -632.0, |
| "logp/rejected": -660.0, |
| "loss": 1.06787109375, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.23828125, |
| "reward/margin": 2.171875, |
| "reward/rejected": -1.9453125, |
| "step": 563 |
| }, |
| { |
| "approx. KL/chosen": 10.625, |
| "approx. KL/rejected": 21.5, |
| "epoch": 0.5778688524590164, |
| "grad_norm": 65.71133422851562, |
| "learning_rate": 4.065570508233807e-06, |
| "logp/chosen": -584.0, |
| "logp/rejected": -704.0, |
| "loss": 1.171875, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 0.427734375, |
| "reward/rejected": -0.9765625, |
| "step": 564 |
| }, |
| { |
| "approx. KL/chosen": 44.5, |
| "approx. KL/rejected": 352.0, |
| "epoch": 0.5788934426229508, |
| "grad_norm": 90.20490264892578, |
| "learning_rate": 4.052747133097854e-06, |
| "logp/chosen": -688.0, |
| "logp/rejected": -1200.0, |
| "loss": 1.3349609375, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.52734375, |
| "reward/margin": 3.421875, |
| "reward/rejected": -3.953125, |
| "step": 565 |
| }, |
| { |
| "approx. KL/chosen": 26.375, |
| "approx. KL/rejected": 81.5, |
| "epoch": 0.5799180327868853, |
| "grad_norm": 67.23853302001953, |
| "learning_rate": 4.0399282038250524e-06, |
| "logp/chosen": -1176.0, |
| "logp/rejected": -1312.0, |
| "loss": 1.000244140625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.11279296875, |
| "reward/margin": 1.984375, |
| "reward/rejected": -1.8671875, |
| "step": 566 |
| }, |
| { |
| "approx. KL/chosen": 36.5, |
| "approx. KL/rejected": 58.25, |
| "epoch": 0.5809426229508197, |
| "grad_norm": 65.2707748413086, |
| "learning_rate": 4.027113884535581e-06, |
| "logp/chosen": -856.0, |
| "logp/rejected": -896.0, |
| "loss": 1.2099609375, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.1240234375, |
| "reward/margin": 1.328125, |
| "reward/rejected": -1.453125, |
| "step": 567 |
| }, |
| { |
| "approx. KL/chosen": 46.25, |
| "approx. KL/rejected": 210.0, |
| "epoch": 0.5819672131147541, |
| "grad_norm": 96.09547424316406, |
| "learning_rate": 4.014304339290612e-06, |
| "logp/chosen": -972.0, |
| "logp/rejected": -1304.0, |
| "loss": 1.10546875, |
| "nll_loss": 0.64453125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.3515625, |
| "reward/margin": 3.546875, |
| "reward/rejected": -3.203125, |
| "step": 568 |
| }, |
| { |
| "approx. KL/chosen": 25.75, |
| "approx. KL/rejected": 79.0, |
| "epoch": 0.5829918032786885, |
| "grad_norm": 81.5541000366211, |
| "learning_rate": 4.0014997320901845e-06, |
| "logp/chosen": -660.0, |
| "logp/rejected": -780.0, |
| "loss": 1.52880859375, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.2255859375, |
| "reward/margin": 0.921875, |
| "reward/rejected": -1.1484375, |
| "step": 569 |
| }, |
| { |
| "approx. KL/chosen": 22.25, |
| "approx. KL/rejected": 326.0, |
| "epoch": 0.5840163934426229, |
| "grad_norm": 19.142375946044922, |
| "learning_rate": 3.988700226871121e-06, |
| "logp/chosen": -1120.0, |
| "logp/rejected": -1608.0, |
| "loss": 0.68896484375, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.55078125, |
| "reward/margin": 4.8125, |
| "reward/rejected": -4.25, |
| "step": 570 |
| }, |
| { |
| "approx. KL/chosen": 83.5, |
| "approx. KL/rejected": 120.0, |
| "epoch": 0.5850409836065574, |
| "grad_norm": 100.134033203125, |
| "learning_rate": 3.975905987504922e-06, |
| "logp/chosen": -840.0, |
| "logp/rejected": -980.0, |
| "loss": 2.05517578125, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.2265625, |
| "reward/margin": 0.8125, |
| "reward/rejected": -2.046875, |
| "step": 571 |
| }, |
| { |
| "approx. KL/chosen": 32.75, |
| "approx. KL/rejected": 211.0, |
| "epoch": 0.5860655737704918, |
| "grad_norm": 110.0105972290039, |
| "learning_rate": 3.963117177795672e-06, |
| "logp/chosen": -864.0, |
| "logp/rejected": -1144.0, |
| "loss": 1.36083984375, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": 0.474609375, |
| "reward/margin": 2.1875, |
| "reward/rejected": -1.71875, |
| "step": 572 |
| }, |
| { |
| "approx. KL/chosen": 157.0, |
| "approx. KL/rejected": 182.0, |
| "epoch": 0.5870901639344263, |
| "grad_norm": 135.29376220703125, |
| "learning_rate": 3.950333961477935e-06, |
| "logp/chosen": -1304.0, |
| "logp/rejected": -1544.0, |
| "loss": 1.33251953125, |
| "nll_loss": 0.515625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": 0.9296875, |
| "reward/margin": 2.5, |
| "reward/rejected": -1.578125, |
| "step": 573 |
| }, |
| { |
| "approx. KL/chosen": 44.25, |
| "approx. KL/rejected": 197.0, |
| "epoch": 0.5881147540983607, |
| "grad_norm": 80.00222778320312, |
| "learning_rate": 3.93755650221467e-06, |
| "logp/chosen": -836.0, |
| "logp/rejected": -1296.0, |
| "loss": 1.09912109375, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.25, |
| "reward/margin": 2.109375, |
| "reward/rejected": -3.359375, |
| "step": 574 |
| }, |
| { |
| "approx. KL/chosen": 106.0, |
| "approx. KL/rejected": 102.5, |
| "epoch": 0.5891393442622951, |
| "grad_norm": 74.1199722290039, |
| "learning_rate": 3.92478496359512e-06, |
| "logp/chosen": -824.0, |
| "logp/rejected": -836.0, |
| "loss": 1.346923828125, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.0, |
| "reward/margin": 0.8984375, |
| "reward/rejected": -1.8984375, |
| "step": 575 |
| }, |
| { |
| "approx. KL/chosen": 107.0, |
| "approx. KL/rejected": 280.0, |
| "epoch": 0.5901639344262295, |
| "grad_norm": 79.17627716064453, |
| "learning_rate": 3.912019509132735e-06, |
| "logp/chosen": -976.0, |
| "logp/rejected": -1312.0, |
| "loss": 1.17626953125, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.69921875, |
| "reward/margin": 4.0625, |
| "reward/rejected": -3.375, |
| "step": 576 |
| }, |
| { |
| "approx. KL/chosen": 105.0, |
| "approx. KL/rejected": 146.0, |
| "epoch": 0.5911885245901639, |
| "grad_norm": 82.53915405273438, |
| "learning_rate": 3.8992603022630625e-06, |
| "logp/chosen": -1520.0, |
| "logp/rejected": -1680.0, |
| "loss": 1.009765625, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.349609375, |
| "reward/margin": 2.890625, |
| "reward/rejected": -2.546875, |
| "step": 577 |
| }, |
| { |
| "approx. KL/chosen": 43.0, |
| "approx. KL/rejected": 98.5, |
| "epoch": 0.5922131147540983, |
| "grad_norm": 117.29817962646484, |
| "learning_rate": 3.886507506341668e-06, |
| "logp/chosen": -956.0, |
| "logp/rejected": -1032.0, |
| "loss": 1.60302734375, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.224609375, |
| "reward/margin": 1.515625, |
| "reward/rejected": -1.7421875, |
| "step": 578 |
| }, |
| { |
| "approx. KL/chosen": 105.0, |
| "approx. KL/rejected": 408.0, |
| "epoch": 0.5932377049180327, |
| "grad_norm": 73.26595306396484, |
| "learning_rate": 3.873761284642036e-06, |
| "logp/chosen": -1416.0, |
| "logp/rejected": -2224.0, |
| "loss": 0.8828125, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 3.8125, |
| "reward/rejected": -4.90625, |
| "step": 579 |
| }, |
| { |
| "approx. KL/chosen": 74.0, |
| "approx. KL/rejected": 206.0, |
| "epoch": 0.5942622950819673, |
| "grad_norm": 92.02568054199219, |
| "learning_rate": 3.861021800353478e-06, |
| "logp/chosen": -1080.0, |
| "logp/rejected": -1704.0, |
| "loss": 1.5126953125, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.3984375, |
| "reward/margin": 3.0, |
| "reward/rejected": -2.609375, |
| "step": 580 |
| }, |
| { |
| "approx. KL/chosen": 56.5, |
| "approx. KL/rejected": 255.0, |
| "epoch": 0.5952868852459017, |
| "grad_norm": 97.84310913085938, |
| "learning_rate": 3.848289216579054e-06, |
| "logp/chosen": -1216.0, |
| "logp/rejected": -1488.0, |
| "loss": 1.22412109375, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.125, |
| "reward/margin": 2.546875, |
| "reward/rejected": -3.671875, |
| "step": 581 |
| }, |
| { |
| "approx. KL/chosen": 19.5, |
| "approx. KL/rejected": 43.25, |
| "epoch": 0.5963114754098361, |
| "grad_norm": 44.318843841552734, |
| "learning_rate": 3.835563696333468e-06, |
| "logp/chosen": -700.0, |
| "logp/rejected": -932.0, |
| "loss": 0.91015625, |
| "nll_loss": 0.48828125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.498046875, |
| "reward/margin": 0.96875, |
| "reward/rejected": -1.46875, |
| "step": 582 |
| }, |
| { |
| "approx. KL/chosen": 37.75, |
| "approx. KL/rejected": 202.0, |
| "epoch": 0.5973360655737705, |
| "grad_norm": 44.31660461425781, |
| "learning_rate": 3.8228454025409985e-06, |
| "logp/chosen": -944.0, |
| "logp/rejected": -1208.0, |
| "loss": 0.93310546875, |
| "nll_loss": 0.66015625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.125, |
| "reward/margin": 3.34375, |
| "reward/rejected": -3.234375, |
| "step": 583 |
| }, |
| { |
| "approx. KL/chosen": 5.5, |
| "approx. KL/rejected": 18.375, |
| "epoch": 0.5983606557377049, |
| "grad_norm": 79.68145751953125, |
| "learning_rate": 3.8101344980334e-06, |
| "logp/chosen": -724.0, |
| "logp/rejected": -772.0, |
| "loss": 1.24169921875, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 0.48828125, |
| "reward/rejected": -0.890625, |
| "step": 584 |
| }, |
| { |
| "approx. KL/chosen": 24.25, |
| "approx. KL/rejected": 140.0, |
| "epoch": 0.5993852459016393, |
| "grad_norm": 29.238313674926758, |
| "learning_rate": 3.7974311455478195e-06, |
| "logp/chosen": -880.0, |
| "logp/rejected": -1176.0, |
| "loss": 0.697509765625, |
| "nll_loss": 0.478515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.04931640625, |
| "reward/margin": 2.984375, |
| "reward/rejected": -2.9375, |
| "step": 585 |
| }, |
| { |
| "approx. KL/chosen": 19.75, |
| "approx. KL/rejected": 158.0, |
| "epoch": 0.6004098360655737, |
| "grad_norm": 71.73645782470703, |
| "learning_rate": 3.7847355077247228e-06, |
| "logp/chosen": -804.0, |
| "logp/rejected": -1184.0, |
| "loss": 1.21533203125, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.0185546875, |
| "reward/margin": 2.1875, |
| "reward/rejected": -2.203125, |
| "step": 586 |
| }, |
| { |
| "epoch": 0.6004098360655737, |
| "eval_approx. KL/chosen": 52.565, |
| "eval_approx. KL/rejected": 181.44, |
| "eval_logp/chosen": -966.72, |
| "eval_logp/rejected": -1289.28, |
| "eval_loss": 1.3400390148162842, |
| "eval_nll_loss": 0.553984375, |
| "eval_reward/accuracy": 0.655, |
| "eval_reward/chosen": -0.506875, |
| "eval_reward/margin": 2.002578125, |
| "eval_reward/rejected": -2.50921875, |
| "eval_runtime": 47.1482, |
| "eval_samples_per_second": 4.2, |
| "eval_steps_per_second": 0.53, |
| "step": 586 |
| }, |
| { |
| "approx. KL/chosen": 34.0, |
| "approx. KL/rejected": 118.0, |
| "epoch": 0.6014344262295082, |
| "grad_norm": 60.84828567504883, |
| "learning_rate": 3.772047747105798e-06, |
| "logp/chosen": -804.0, |
| "logp/rejected": -1072.0, |
| "loss": 1.263671875, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.0234375, |
| "reward/margin": 0.81640625, |
| "reward/rejected": -1.8359375, |
| "step": 587 |
| }, |
| { |
| "approx. KL/chosen": 26.75, |
| "approx. KL/rejected": 324.0, |
| "epoch": 0.6024590163934426, |
| "grad_norm": 10.29672908782959, |
| "learning_rate": 3.759368026131888e-06, |
| "logp/chosen": -976.0, |
| "logp/rejected": -1976.0, |
| "loss": 0.7236328125, |
| "nll_loss": 0.6640625, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.474609375, |
| "reward/margin": 4.3125, |
| "reward/rejected": -4.78125, |
| "step": 588 |
| }, |
| { |
| "approx. KL/chosen": 183.0, |
| "approx. KL/rejected": 165.0, |
| "epoch": 0.6034836065573771, |
| "grad_norm": 154.64537048339844, |
| "learning_rate": 3.7466965071409e-06, |
| "logp/chosen": -1312.0, |
| "logp/rejected": -1584.0, |
| "loss": 2.151611328125, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -2.109375, |
| "reward/margin": 0.3984375, |
| "reward/rejected": -2.5, |
| "step": 589 |
| }, |
| { |
| "approx. KL/chosen": 20.0, |
| "approx. KL/rejected": 52.25, |
| "epoch": 0.6045081967213115, |
| "grad_norm": 113.6096420288086, |
| "learning_rate": 3.734033352365731e-06, |
| "logp/chosen": -968.0, |
| "logp/rejected": -1176.0, |
| "loss": 1.7197265625, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 0.44921875, |
| "reward/rejected": -0.8515625, |
| "step": 590 |
| }, |
| { |
| "approx. KL/chosen": 99.0, |
| "approx. KL/rejected": 120.0, |
| "epoch": 0.6055327868852459, |
| "grad_norm": 99.07518005371094, |
| "learning_rate": 3.721378723932198e-06, |
| "logp/chosen": -896.0, |
| "logp/rejected": -1104.0, |
| "loss": 1.873046875, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 1.609375, |
| "reward/rejected": -2.15625, |
| "step": 591 |
| }, |
| { |
| "approx. KL/chosen": 46.5, |
| "approx. KL/rejected": 166.0, |
| "epoch": 0.6065573770491803, |
| "grad_norm": 63.52478790283203, |
| "learning_rate": 3.7087327838569477e-06, |
| "logp/chosen": -716.0, |
| "logp/rejected": -956.0, |
| "loss": 0.822998046875, |
| "nll_loss": 0.37109375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.171875, |
| "reward/margin": 1.5546875, |
| "reward/rejected": -2.734375, |
| "step": 592 |
| }, |
| { |
| "approx. KL/chosen": 105.5, |
| "approx. KL/rejected": 274.0, |
| "epoch": 0.6075819672131147, |
| "grad_norm": 170.55935668945312, |
| "learning_rate": 3.696095694045394e-06, |
| "logp/chosen": -1736.0, |
| "logp/rejected": -2320.0, |
| "loss": 2.09326171875, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.5234375, |
| "reward/margin": 2.6875, |
| "reward/rejected": -3.203125, |
| "step": 593 |
| }, |
| { |
| "approx. KL/chosen": 7.0, |
| "approx. KL/rejected": 131.0, |
| "epoch": 0.6086065573770492, |
| "grad_norm": 60.4393424987793, |
| "learning_rate": 3.6834676162896415e-06, |
| "logp/chosen": -564.0, |
| "logp/rejected": -696.0, |
| "loss": 1.011962890625, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 1.9375, |
| "reward/rejected": -2.03125, |
| "step": 594 |
| }, |
| { |
| "approx. KL/chosen": 90.0, |
| "approx. KL/rejected": 160.0, |
| "epoch": 0.6096311475409836, |
| "grad_norm": 91.24703216552734, |
| "learning_rate": 3.6708487122664124e-06, |
| "logp/chosen": -992.0, |
| "logp/rejected": -1256.0, |
| "loss": 1.416259765625, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 2.5625, |
| "reward/rejected": -3.21875, |
| "step": 595 |
| }, |
| { |
| "approx. KL/chosen": 18.0, |
| "approx. KL/rejected": 394.0, |
| "epoch": 0.610655737704918, |
| "grad_norm": 25.625228881835938, |
| "learning_rate": 3.6582391435349786e-06, |
| "logp/chosen": -620.0, |
| "logp/rejected": -1184.0, |
| "loss": 0.74169921875, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.025634765625, |
| "reward/margin": 4.5625, |
| "reward/rejected": -4.59375, |
| "step": 596 |
| }, |
| { |
| "approx. KL/chosen": 77.0, |
| "approx. KL/rejected": 98.5, |
| "epoch": 0.6116803278688525, |
| "grad_norm": 120.1687240600586, |
| "learning_rate": 3.6456390715350934e-06, |
| "logp/chosen": -728.0, |
| "logp/rejected": -772.0, |
| "loss": 1.71533203125, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.94921875, |
| "reward/margin": 1.0625, |
| "reward/rejected": -2.015625, |
| "step": 597 |
| }, |
| { |
| "approx. KL/chosen": 46.0, |
| "approx. KL/rejected": 107.0, |
| "epoch": 0.6127049180327869, |
| "grad_norm": 127.66960906982422, |
| "learning_rate": 3.633048657584922e-06, |
| "logp/chosen": -1160.0, |
| "logp/rejected": -1400.0, |
| "loss": 1.7900390625, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 0.6484375, |
| "reward/rejected": -1.453125, |
| "step": 598 |
| }, |
| { |
| "approx. KL/chosen": 110.0, |
| "approx. KL/rejected": 187.0, |
| "epoch": 0.6137295081967213, |
| "grad_norm": 120.04248046875, |
| "learning_rate": 3.6204680628789808e-06, |
| "logp/chosen": -1200.0, |
| "logp/rejected": -1280.0, |
| "loss": 1.57080078125, |
| "nll_loss": 0.5, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -2.40625, |
| "reward/margin": 0.94921875, |
| "reward/rejected": -3.359375, |
| "step": 599 |
| }, |
| { |
| "approx. KL/chosen": 57.25, |
| "approx. KL/rejected": 334.0, |
| "epoch": 0.6147540983606558, |
| "grad_norm": 77.03280639648438, |
| "learning_rate": 3.607897448486068e-06, |
| "logp/chosen": -980.0, |
| "logp/rejected": -1392.0, |
| "loss": 1.03857421875, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.3515625, |
| "reward/margin": 2.84375, |
| "reward/rejected": -4.1875, |
| "step": 600 |
| }, |
| { |
| "approx. KL/chosen": 68.0, |
| "approx. KL/rejected": 157.0, |
| "epoch": 0.6157786885245902, |
| "grad_norm": 99.64236450195312, |
| "learning_rate": 3.595336975347206e-06, |
| "logp/chosen": -1368.0, |
| "logp/rejected": -1656.0, |
| "loss": 1.05908203125, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.3984375, |
| "reward/margin": 1.90625, |
| "reward/rejected": -3.296875, |
| "step": 601 |
| }, |
| { |
| "approx. KL/chosen": 18.25, |
| "approx. KL/rejected": 136.0, |
| "epoch": 0.6168032786885246, |
| "grad_norm": 43.99106979370117, |
| "learning_rate": 3.5827868042735787e-06, |
| "logp/chosen": -536.0, |
| "logp/rejected": -616.0, |
| "loss": 0.858642578125, |
| "nll_loss": 0.46484375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.67578125, |
| "reward/margin": 2.125, |
| "reward/rejected": -2.796875, |
| "step": 602 |
| }, |
| { |
| "approx. KL/chosen": 36.25, |
| "approx. KL/rejected": 87.5, |
| "epoch": 0.617827868852459, |
| "grad_norm": 106.52737426757812, |
| "learning_rate": 3.570247095944474e-06, |
| "logp/chosen": -816.0, |
| "logp/rejected": -808.0, |
| "loss": 1.60302734375, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.25, |
| "reward/margin": 0.451171875, |
| "reward/rejected": -1.703125, |
| "step": 603 |
| }, |
| { |
| "approx. KL/chosen": 30.75, |
| "approx. KL/rejected": 199.0, |
| "epoch": 0.6188524590163934, |
| "grad_norm": 33.587188720703125, |
| "learning_rate": 3.5577180109052226e-06, |
| "logp/chosen": -856.0, |
| "logp/rejected": -1136.0, |
| "loss": 0.92236328125, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.11279296875, |
| "reward/margin": 2.90625, |
| "reward/rejected": -2.796875, |
| "step": 604 |
| }, |
| { |
| "approx. KL/chosen": 156.0, |
| "approx. KL/rejected": 87.5, |
| "epoch": 0.6198770491803278, |
| "grad_norm": 156.4219207763672, |
| "learning_rate": 3.5451997095651506e-06, |
| "logp/chosen": -1696.0, |
| "logp/rejected": -1440.0, |
| "loss": 3.08056640625, |
| "nll_loss": 0.640625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0, |
| "reward/margin": -0.0400390625, |
| "reward/rejected": -0.9609375, |
| "step": 605 |
| }, |
| { |
| "approx. KL/chosen": 73.5, |
| "approx. KL/rejected": 57.5, |
| "epoch": 0.6209016393442623, |
| "grad_norm": 109.8095932006836, |
| "learning_rate": 3.5326923521955162e-06, |
| "logp/chosen": -840.0, |
| "logp/rejected": -872.0, |
| "loss": 1.86669921875, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.3046875, |
| "reward/margin": 0.011962890625, |
| "reward/rejected": -1.3125, |
| "step": 606 |
| }, |
| { |
| "approx. KL/chosen": 49.0, |
| "approx. KL/rejected": 298.0, |
| "epoch": 0.6219262295081968, |
| "grad_norm": 97.95024108886719, |
| "learning_rate": 3.5201960989274666e-06, |
| "logp/chosen": -1200.0, |
| "logp/rejected": -1456.0, |
| "loss": 1.2265625, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 3.6875, |
| "reward/rejected": -3.984375, |
| "step": 607 |
| }, |
| { |
| "approx. KL/chosen": 35.25, |
| "approx. KL/rejected": 266.0, |
| "epoch": 0.6229508196721312, |
| "grad_norm": 43.69232177734375, |
| "learning_rate": 3.5077111097499785e-06, |
| "logp/chosen": -916.0, |
| "logp/rejected": -1512.0, |
| "loss": 0.742431640625, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.25, |
| "reward/margin": 3.90625, |
| "reward/rejected": -4.15625, |
| "step": 608 |
| }, |
| { |
| "approx. KL/chosen": 41.5, |
| "approx. KL/rejected": 166.0, |
| "epoch": 0.6239754098360656, |
| "grad_norm": 93.61070251464844, |
| "learning_rate": 3.495237544507821e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1272.0, |
| "loss": 1.003662109375, |
| "nll_loss": 0.4921875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.0, |
| "reward/margin": 3.0625, |
| "reward/rejected": -3.0625, |
| "step": 609 |
| }, |
| { |
| "approx. KL/chosen": 24.25, |
| "approx. KL/rejected": 69.0, |
| "epoch": 0.625, |
| "grad_norm": 51.61826705932617, |
| "learning_rate": 3.4827755628994977e-06, |
| "logp/chosen": -848.0, |
| "logp/rejected": -996.0, |
| "loss": 0.94384765625, |
| "nll_loss": 0.515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.138671875, |
| "reward/margin": 1.3671875, |
| "reward/rejected": -1.5, |
| "step": 610 |
| }, |
| { |
| "approx. KL/chosen": 28.0, |
| "approx. KL/rejected": 209.0, |
| "epoch": 0.6260245901639344, |
| "grad_norm": 54.6675910949707, |
| "learning_rate": 3.4703253244752126e-06, |
| "logp/chosen": -736.0, |
| "logp/rejected": -976.0, |
| "loss": 1.060546875, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 2.5625, |
| "reward/rejected": -2.75, |
| "step": 611 |
| }, |
| { |
| "approx. KL/chosen": 25.25, |
| "approx. KL/rejected": 358.0, |
| "epoch": 0.6270491803278688, |
| "grad_norm": 40.20774841308594, |
| "learning_rate": 3.457886988634816e-06, |
| "logp/chosen": -880.0, |
| "logp/rejected": -1224.0, |
| "loss": 0.72021484375, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.94921875, |
| "reward/margin": 3.90625, |
| "reward/rejected": -4.84375, |
| "step": 612 |
| }, |
| { |
| "approx. KL/chosen": 13.3125, |
| "approx. KL/rejected": 192.0, |
| "epoch": 0.6280737704918032, |
| "grad_norm": 69.31623840332031, |
| "learning_rate": 3.445460714625777e-06, |
| "logp/chosen": -952.0, |
| "logp/rejected": -1312.0, |
| "loss": 0.97216796875, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.32421875, |
| "reward/margin": 2.359375, |
| "reward/rejected": -2.671875, |
| "step": 613 |
| }, |
| { |
| "approx. KL/chosen": 54.0, |
| "approx. KL/rejected": 86.5, |
| "epoch": 0.6290983606557377, |
| "grad_norm": 105.49286651611328, |
| "learning_rate": 3.4330466615411293e-06, |
| "logp/chosen": -1064.0, |
| "logp/rejected": -1280.0, |
| "loss": 1.383544921875, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 1.078125, |
| "reward/rejected": -1.6796875, |
| "step": 614 |
| }, |
| { |
| "approx. KL/chosen": 59.5, |
| "approx. KL/rejected": 314.0, |
| "epoch": 0.6301229508196722, |
| "grad_norm": 34.788047790527344, |
| "learning_rate": 3.4206449883174528e-06, |
| "logp/chosen": -1248.0, |
| "logp/rejected": -1696.0, |
| "loss": 0.82958984375, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.125, |
| "reward/margin": 3.71875, |
| "reward/rejected": -3.84375, |
| "step": 615 |
| }, |
| { |
| "approx. KL/chosen": 108.0, |
| "approx. KL/rejected": 98.0, |
| "epoch": 0.6311475409836066, |
| "grad_norm": 127.1442642211914, |
| "learning_rate": 3.408255853732819e-06, |
| "logp/chosen": -1160.0, |
| "logp/rejected": -1104.0, |
| "loss": 1.9033203125, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.5, |
| "reward/margin": 0.427734375, |
| "reward/rejected": -1.9296875, |
| "step": 616 |
| }, |
| { |
| "approx. KL/chosen": 20.5, |
| "approx. KL/rejected": 86.0, |
| "epoch": 0.632172131147541, |
| "grad_norm": 85.9444580078125, |
| "learning_rate": 3.3958794164047744e-06, |
| "logp/chosen": -828.0, |
| "logp/rejected": -868.0, |
| "loss": 1.3193359375, |
| "nll_loss": 0.66796875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 0.6484375, |
| "reward/rejected": -1.25, |
| "step": 617 |
| }, |
| { |
| "approx. KL/chosen": 47.25, |
| "approx. KL/rejected": 182.0, |
| "epoch": 0.6331967213114754, |
| "grad_norm": 64.75737762451172, |
| "learning_rate": 3.3835158347882988e-06, |
| "logp/chosen": -1088.0, |
| "logp/rejected": -1416.0, |
| "loss": 0.87109375, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.046875, |
| "reward/margin": 2.359375, |
| "reward/rejected": -3.40625, |
| "step": 618 |
| }, |
| { |
| "approx. KL/chosen": 60.0, |
| "approx. KL/rejected": 197.0, |
| "epoch": 0.6342213114754098, |
| "grad_norm": 119.64144897460938, |
| "learning_rate": 3.371165267173784e-06, |
| "logp/chosen": -1360.0, |
| "logp/rejected": -1704.0, |
| "loss": 1.253173828125, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 2.3125, |
| "reward/rejected": -2.90625, |
| "step": 619 |
| }, |
| { |
| "approx. KL/chosen": 75.0, |
| "approx. KL/rejected": 230.0, |
| "epoch": 0.6352459016393442, |
| "grad_norm": 134.3433837890625, |
| "learning_rate": 3.358827871685e-06, |
| "logp/chosen": -1064.0, |
| "logp/rejected": -1304.0, |
| "loss": 2.05029296875, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.1484375, |
| "reward/margin": 1.390625, |
| "reward/rejected": -2.546875, |
| "step": 620 |
| }, |
| { |
| "approx. KL/chosen": 26.75, |
| "approx. KL/rejected": 172.0, |
| "epoch": 0.6362704918032787, |
| "grad_norm": 26.70509910583496, |
| "learning_rate": 3.3465038062770824e-06, |
| "logp/chosen": -744.0, |
| "logp/rejected": -1064.0, |
| "loss": 0.776611328125, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.17578125, |
| "reward/margin": 2.875, |
| "reward/rejected": -3.046875, |
| "step": 621 |
| }, |
| { |
| "approx. KL/chosen": 88.0, |
| "approx. KL/rejected": 116.0, |
| "epoch": 0.6372950819672131, |
| "grad_norm": 89.1851806640625, |
| "learning_rate": 3.3341932287344917e-06, |
| "logp/chosen": -816.0, |
| "logp/rejected": -776.0, |
| "loss": 1.98681640625, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0546875, |
| "reward/margin": 0.6640625, |
| "reward/rejected": -1.7109375, |
| "step": 622 |
| }, |
| { |
| "approx. KL/chosen": 17.75, |
| "approx. KL/rejected": 744.0, |
| "epoch": 0.6383196721311475, |
| "grad_norm": 27.031137466430664, |
| "learning_rate": 3.321896296669014e-06, |
| "logp/chosen": -856.0, |
| "logp/rejected": -1408.0, |
| "loss": 0.65185546875, |
| "nll_loss": 0.52734375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.42578125, |
| "reward/margin": 5.53125, |
| "reward/rejected": -5.96875, |
| "step": 623 |
| }, |
| { |
| "approx. KL/chosen": 51.5, |
| "approx. KL/rejected": 147.0, |
| "epoch": 0.639344262295082, |
| "grad_norm": 72.03306579589844, |
| "learning_rate": 3.3096131675177265e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -1136.0, |
| "loss": 1.26953125, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 2.296875, |
| "reward/rejected": -3.09375, |
| "step": 624 |
| }, |
| { |
| "approx. KL/chosen": 49.5, |
| "approx. KL/rejected": 164.0, |
| "epoch": 0.6403688524590164, |
| "grad_norm": 93.35977172851562, |
| "learning_rate": 3.297343998540991e-06, |
| "logp/chosen": -932.0, |
| "logp/rejected": -1136.0, |
| "loss": 1.52294921875, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 1.578125, |
| "reward/rejected": -2.171875, |
| "step": 625 |
| }, |
| { |
| "approx. KL/chosen": 13.8125, |
| "approx. KL/rejected": 169.0, |
| "epoch": 0.6413934426229508, |
| "grad_norm": 51.85264587402344, |
| "learning_rate": 3.2850889468204345e-06, |
| "logp/chosen": -1000.0, |
| "logp/rejected": -1344.0, |
| "loss": 1.06884765625, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.875, |
| "reward/margin": 1.96875, |
| "reward/rejected": -2.84375, |
| "step": 626 |
| }, |
| { |
| "approx. KL/chosen": 41.0, |
| "approx. KL/rejected": 182.0, |
| "epoch": 0.6424180327868853, |
| "grad_norm": 92.13611602783203, |
| "learning_rate": 3.272848169256948e-06, |
| "logp/chosen": -968.0, |
| "logp/rejected": -1328.0, |
| "loss": 1.251953125, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.099609375, |
| "reward/margin": 2.09375, |
| "reward/rejected": -2.203125, |
| "step": 627 |
| }, |
| { |
| "approx. KL/chosen": 7.0, |
| "approx. KL/rejected": 412.0, |
| "epoch": 0.6434426229508197, |
| "grad_norm": 71.10479736328125, |
| "learning_rate": 3.260621822568664e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1536.0, |
| "loss": 0.931640625, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 3.609375, |
| "reward/rejected": -3.90625, |
| "step": 628 |
| }, |
| { |
| "approx. KL/chosen": 40.25, |
| "approx. KL/rejected": 233.0, |
| "epoch": 0.6444672131147541, |
| "grad_norm": 99.74188232421875, |
| "learning_rate": 3.248410063288962e-06, |
| "logp/chosen": -1280.0, |
| "logp/rejected": -1752.0, |
| "loss": 1.2470703125, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.050048828125, |
| "reward/margin": 3.046875, |
| "reward/rejected": -3.109375, |
| "step": 629 |
| }, |
| { |
| "approx. KL/chosen": 109.0, |
| "approx. KL/rejected": 113.5, |
| "epoch": 0.6454918032786885, |
| "grad_norm": 73.19153594970703, |
| "learning_rate": 3.2362130477644566e-06, |
| "logp/chosen": -1176.0, |
| "logp/rejected": -1440.0, |
| "loss": 1.059814453125, |
| "nll_loss": 0.515625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.00146484375, |
| "reward/margin": 2.4375, |
| "reward/rejected": -2.421875, |
| "step": 630 |
| }, |
| { |
| "approx. KL/chosen": 24.5, |
| "approx. KL/rejected": 163.0, |
| "epoch": 0.6465163934426229, |
| "grad_norm": 56.31940460205078, |
| "learning_rate": 3.2240309321530008e-06, |
| "logp/chosen": -880.0, |
| "logp/rejected": -1368.0, |
| "loss": 0.79296875, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.17578125, |
| "reward/margin": 3.28125, |
| "reward/rejected": -3.109375, |
| "step": 631 |
| }, |
| { |
| "approx. KL/chosen": 6.0625, |
| "approx. KL/rejected": 34.25, |
| "epoch": 0.6475409836065574, |
| "grad_norm": 64.83807373046875, |
| "learning_rate": 3.2118638724216834e-06, |
| "logp/chosen": -708.0, |
| "logp/rejected": -844.0, |
| "loss": 1.11328125, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.42578125, |
| "reward/margin": 0.6015625, |
| "reward/rejected": -1.03125, |
| "step": 632 |
| }, |
| { |
| "approx. KL/chosen": 16.75, |
| "approx. KL/rejected": 52.5, |
| "epoch": 0.6485655737704918, |
| "grad_norm": 58.36334228515625, |
| "learning_rate": 3.1997120243448335e-06, |
| "logp/chosen": -576.0, |
| "logp/rejected": -624.0, |
| "loss": 1.3349609375, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.953125, |
| "reward/margin": 0.314453125, |
| "reward/rejected": -1.265625, |
| "step": 633 |
| }, |
| { |
| "approx. KL/chosen": 19.0, |
| "approx. KL/rejected": 169.0, |
| "epoch": 0.6495901639344263, |
| "grad_norm": 47.25831604003906, |
| "learning_rate": 3.1875755435020264e-06, |
| "logp/chosen": -600.0, |
| "logp/rejected": -1048.0, |
| "loss": 1.076171875, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.17578125, |
| "reward/margin": 2.5, |
| "reward/rejected": -2.671875, |
| "step": 634 |
| }, |
| { |
| "approx. KL/chosen": 30.25, |
| "approx. KL/rejected": 113.0, |
| "epoch": 0.6506147540983607, |
| "grad_norm": 94.20828247070312, |
| "learning_rate": 3.1754545852760928e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1184.0, |
| "loss": 1.248046875, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.75, |
| "reward/margin": 1.15625, |
| "reward/rejected": -1.90625, |
| "step": 635 |
| }, |
| { |
| "approx. KL/chosen": 43.75, |
| "approx. KL/rejected": 191.0, |
| "epoch": 0.6516393442622951, |
| "grad_norm": 55.33634948730469, |
| "learning_rate": 3.163349304851124e-06, |
| "logp/chosen": -932.0, |
| "logp/rejected": -1224.0, |
| "loss": 0.98193359375, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.55078125, |
| "reward/margin": 3.296875, |
| "reward/rejected": -2.75, |
| "step": 636 |
| }, |
| { |
| "approx. KL/chosen": 21.25, |
| "approx. KL/rejected": 86.5, |
| "epoch": 0.6526639344262295, |
| "grad_norm": 58.36037826538086, |
| "learning_rate": 3.151259857210493e-06, |
| "logp/chosen": -696.0, |
| "logp/rejected": -984.0, |
| "loss": 0.932373046875, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.423828125, |
| "reward/margin": 1.8515625, |
| "reward/rejected": -2.28125, |
| "step": 637 |
| }, |
| { |
| "approx. KL/chosen": 30.25, |
| "approx. KL/rejected": 150.0, |
| "epoch": 0.6536885245901639, |
| "grad_norm": 53.01877212524414, |
| "learning_rate": 3.1391863971348668e-06, |
| "logp/chosen": -1256.0, |
| "logp/rejected": -1240.0, |
| "loss": 0.77734375, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.65234375, |
| "reward/margin": 3.40625, |
| "reward/rejected": -2.765625, |
| "step": 638 |
| }, |
| { |
| "approx. KL/chosen": 43.5, |
| "approx. KL/rejected": 140.0, |
| "epoch": 0.6547131147540983, |
| "grad_norm": 132.91455078125, |
| "learning_rate": 3.127129079200219e-06, |
| "logp/chosen": -904.0, |
| "logp/rejected": -1232.0, |
| "loss": 2.59619140625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.4140625, |
| "reward/margin": -0.80859375, |
| "reward/rejected": -0.6015625, |
| "step": 639 |
| }, |
| { |
| "approx. KL/chosen": 11.0, |
| "approx. KL/rejected": 141.0, |
| "epoch": 0.6557377049180327, |
| "grad_norm": 58.09550476074219, |
| "learning_rate": 3.115088057775863e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1072.0, |
| "loss": 0.885986328125, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 2.15625, |
| "reward/rejected": -2.25, |
| "step": 640 |
| }, |
| { |
| "approx. KL/chosen": 46.25, |
| "approx. KL/rejected": 228.0, |
| "epoch": 0.6567622950819673, |
| "grad_norm": 94.83672332763672, |
| "learning_rate": 3.103063487022462e-06, |
| "logp/chosen": -780.0, |
| "logp/rejected": -1240.0, |
| "loss": 1.122314453125, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.1640625, |
| "reward/margin": 1.8828125, |
| "reward/rejected": -3.046875, |
| "step": 641 |
| }, |
| { |
| "approx. KL/chosen": 18.5, |
| "approx. KL/rejected": 123.0, |
| "epoch": 0.6577868852459017, |
| "grad_norm": 57.03097152709961, |
| "learning_rate": 3.0910555208900685e-06, |
| "logp/chosen": -844.0, |
| "logp/rejected": -960.0, |
| "loss": 0.9384765625, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 2.078125, |
| "reward/rejected": -2.375, |
| "step": 642 |
| }, |
| { |
| "approx. KL/chosen": 27.0, |
| "approx. KL/rejected": 358.0, |
| "epoch": 0.6588114754098361, |
| "grad_norm": 63.84714126586914, |
| "learning_rate": 3.0790643131161417e-06, |
| "logp/chosen": -1012.0, |
| "logp/rejected": -1672.0, |
| "loss": 0.99169921875, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.5, |
| "reward/margin": 3.890625, |
| "reward/rejected": -4.375, |
| "step": 643 |
| }, |
| { |
| "approx. KL/chosen": 64.0, |
| "approx. KL/rejected": 78.5, |
| "epoch": 0.6598360655737705, |
| "grad_norm": 66.11740112304688, |
| "learning_rate": 3.067090017223586e-06, |
| "logp/chosen": -828.0, |
| "logp/rejected": -1008.0, |
| "loss": 1.44921875, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.19921875, |
| "reward/margin": 1.71875, |
| "reward/rejected": -1.9140625, |
| "step": 644 |
| }, |
| { |
| "approx. KL/chosen": 36.75, |
| "approx. KL/rejected": 74.5, |
| "epoch": 0.6608606557377049, |
| "grad_norm": 158.7051544189453, |
| "learning_rate": 3.0551327865187856e-06, |
| "logp/chosen": -1056.0, |
| "logp/rejected": -1104.0, |
| "loss": 1.281494140625, |
| "nll_loss": 0.474609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.07421875, |
| "reward/margin": 1.2265625, |
| "reward/rejected": -1.3046875, |
| "step": 645 |
| }, |
| { |
| "approx. KL/chosen": 17.125, |
| "approx. KL/rejected": 113.0, |
| "epoch": 0.6618852459016393, |
| "grad_norm": 35.46870422363281, |
| "learning_rate": 3.043192774089637e-06, |
| "logp/chosen": -776.0, |
| "logp/rejected": -956.0, |
| "loss": 0.90771484375, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.318359375, |
| "reward/margin": 2.140625, |
| "reward/rejected": -2.46875, |
| "step": 646 |
| }, |
| { |
| "approx. KL/chosen": 38.25, |
| "approx. KL/rejected": 290.0, |
| "epoch": 0.6629098360655737, |
| "grad_norm": 75.65243530273438, |
| "learning_rate": 3.031270132803592e-06, |
| "logp/chosen": -772.0, |
| "logp/rejected": -1168.0, |
| "loss": 1.314453125, |
| "nll_loss": 0.515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.64453125, |
| "reward/margin": 2.546875, |
| "reward/rejected": -3.203125, |
| "step": 647 |
| }, |
| { |
| "approx. KL/chosen": 14.0, |
| "approx. KL/rejected": 106.0, |
| "epoch": 0.6639344262295082, |
| "grad_norm": 80.30498504638672, |
| "learning_rate": 3.0193650153057036e-06, |
| "logp/chosen": -912.0, |
| "logp/rejected": -904.0, |
| "loss": 1.0693359375, |
| "nll_loss": 0.4921875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 1.15625, |
| "reward/rejected": -1.75, |
| "step": 648 |
| }, |
| { |
| "approx. KL/chosen": 43.5, |
| "approx. KL/rejected": 306.0, |
| "epoch": 0.6649590163934426, |
| "grad_norm": 72.28192901611328, |
| "learning_rate": 3.0074775740166647e-06, |
| "logp/chosen": -848.0, |
| "logp/rejected": -1040.0, |
| "loss": 1.115234375, |
| "nll_loss": 0.6875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 3.890625, |
| "reward/rejected": -4.1875, |
| "step": 649 |
| }, |
| { |
| "approx. KL/chosen": 35.0, |
| "approx. KL/rejected": 322.0, |
| "epoch": 0.6659836065573771, |
| "grad_norm": 61.026039123535156, |
| "learning_rate": 2.9956079611308625e-06, |
| "logp/chosen": -1136.0, |
| "logp/rejected": -1632.0, |
| "loss": 0.82080078125, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.82421875, |
| "reward/margin": 3.453125, |
| "reward/rejected": -4.28125, |
| "step": 650 |
| }, |
| { |
| "approx. KL/chosen": 27.0, |
| "approx. KL/rejected": 274.0, |
| "epoch": 0.6670081967213115, |
| "grad_norm": 65.25059509277344, |
| "learning_rate": 2.9837563286144276e-06, |
| "logp/chosen": -988.0, |
| "logp/rejected": -1296.0, |
| "loss": 1.109375, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.703125, |
| "reward/margin": 2.703125, |
| "reward/rejected": -3.40625, |
| "step": 651 |
| }, |
| { |
| "approx. KL/chosen": 152.0, |
| "approx. KL/rejected": 59.75, |
| "epoch": 0.6680327868852459, |
| "grad_norm": 127.27412414550781, |
| "learning_rate": 2.9719228282032893e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1360.0, |
| "loss": 1.881591796875, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.2890625, |
| "reward/margin": 0.54296875, |
| "reward/rejected": -1.828125, |
| "step": 652 |
| }, |
| { |
| "approx. KL/chosen": 27.25, |
| "approx. KL/rejected": 82.5, |
| "epoch": 0.6690573770491803, |
| "grad_norm": 62.2140007019043, |
| "learning_rate": 2.9601076114012306e-06, |
| "logp/chosen": -788.0, |
| "logp/rejected": -800.0, |
| "loss": 0.89990234375, |
| "nll_loss": 0.46875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.7265625, |
| "reward/margin": 1.4296875, |
| "reward/rejected": -2.15625, |
| "step": 653 |
| }, |
| { |
| "approx. KL/chosen": 129.0, |
| "approx. KL/rejected": 194.0, |
| "epoch": 0.6700819672131147, |
| "grad_norm": 114.2888412475586, |
| "learning_rate": 2.9483108294779523e-06, |
| "logp/chosen": -1088.0, |
| "logp/rejected": -1256.0, |
| "loss": 2.05029296875, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -2.53125, |
| "reward/margin": 0.173828125, |
| "reward/rejected": -2.703125, |
| "step": 654 |
| }, |
| { |
| "approx. KL/chosen": 13.125, |
| "approx. KL/rejected": 382.0, |
| "epoch": 0.6711065573770492, |
| "grad_norm": 25.0899600982666, |
| "learning_rate": 2.9365326334671326e-06, |
| "logp/chosen": -592.0, |
| "logp/rejected": -1016.0, |
| "loss": 0.719482421875, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.451171875, |
| "reward/margin": 4.6875, |
| "reward/rejected": -4.21875, |
| "step": 655 |
| }, |
| { |
| "approx. KL/chosen": 50.0, |
| "approx. KL/rejected": 636.0, |
| "epoch": 0.6721311475409836, |
| "grad_norm": 65.18437957763672, |
| "learning_rate": 2.924773174164498e-06, |
| "logp/chosen": -1216.0, |
| "logp/rejected": -1848.0, |
| "loss": 1.04443359375, |
| "nll_loss": 0.66015625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.8046875, |
| "reward/margin": 4.125, |
| "reward/rejected": -5.9375, |
| "step": 656 |
| }, |
| { |
| "approx. KL/chosen": 54.5, |
| "approx. KL/rejected": 106.0, |
| "epoch": 0.673155737704918, |
| "grad_norm": 88.5308837890625, |
| "learning_rate": 2.9130326021258837e-06, |
| "logp/chosen": -984.0, |
| "logp/rejected": -1208.0, |
| "loss": 1.480224609375, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.6640625, |
| "reward/margin": 1.546875, |
| "reward/rejected": -2.21875, |
| "step": 657 |
| }, |
| { |
| "approx. KL/chosen": 24.5, |
| "approx. KL/rejected": 92.5, |
| "epoch": 0.6741803278688525, |
| "grad_norm": 57.663002014160156, |
| "learning_rate": 2.90131106766532e-06, |
| "logp/chosen": -728.0, |
| "logp/rejected": -864.0, |
| "loss": 1.2109375, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 1.5, |
| "reward/rejected": -1.8046875, |
| "step": 658 |
| }, |
| { |
| "approx. KL/chosen": 61.25, |
| "approx. KL/rejected": 70.0, |
| "epoch": 0.6752049180327869, |
| "grad_norm": 120.48323059082031, |
| "learning_rate": 2.8896087208530927e-06, |
| "logp/chosen": -792.0, |
| "logp/rejected": -732.0, |
| "loss": 1.70751953125, |
| "nll_loss": 0.4765625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.671875, |
| "reward/margin": 0.150390625, |
| "reward/rejected": -1.828125, |
| "step": 659 |
| }, |
| { |
| "approx. KL/chosen": 42.75, |
| "approx. KL/rejected": 55.75, |
| "epoch": 0.6762295081967213, |
| "grad_norm": 118.77835845947266, |
| "learning_rate": 2.8779257115138337e-06, |
| "logp/chosen": -1160.0, |
| "logp/rejected": -960.0, |
| "loss": 1.94775390625, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": 0.150390625, |
| "reward/margin": 0.2265625, |
| "reward/rejected": -0.0751953125, |
| "step": 660 |
| }, |
| { |
| "approx. KL/chosen": 78.5, |
| "approx. KL/rejected": 92.0, |
| "epoch": 0.6772540983606558, |
| "grad_norm": 70.39257049560547, |
| "learning_rate": 2.866262189224596e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -1352.0, |
| "loss": 1.3681640625, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.173828125, |
| "reward/margin": 1.578125, |
| "reward/rejected": -1.75, |
| "step": 661 |
| }, |
| { |
| "approx. KL/chosen": 14.5, |
| "approx. KL/rejected": 166.0, |
| "epoch": 0.6782786885245902, |
| "grad_norm": 39.48624038696289, |
| "learning_rate": 2.8546183033129437e-06, |
| "logp/chosen": -1020.0, |
| "logp/rejected": -1584.0, |
| "loss": 0.95166015625, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.0, |
| "reward/margin": 2.65625, |
| "reward/rejected": -2.65625, |
| "step": 662 |
| }, |
| { |
| "approx. KL/chosen": 36.25, |
| "approx. KL/rejected": 236.0, |
| "epoch": 0.6793032786885246, |
| "grad_norm": 81.6996078491211, |
| "learning_rate": 2.842994202855031e-06, |
| "logp/chosen": -912.0, |
| "logp/rejected": -1232.0, |
| "loss": 1.336181640625, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": 0.75, |
| "reward/margin": 2.5, |
| "reward/rejected": -1.75, |
| "step": 663 |
| }, |
| { |
| "approx. KL/chosen": 3.1875, |
| "approx. KL/rejected": 56.0, |
| "epoch": 0.680327868852459, |
| "grad_norm": 67.76034545898438, |
| "learning_rate": 2.8313900366737063e-06, |
| "logp/chosen": -824.0, |
| "logp/rejected": -712.0, |
| "loss": 1.06982421875, |
| "nll_loss": 0.48046875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.287109375, |
| "reward/margin": 1.0390625, |
| "reward/rejected": -1.328125, |
| "step": 664 |
| }, |
| { |
| "approx. KL/chosen": 31.5, |
| "approx. KL/rejected": 88.5, |
| "epoch": 0.6813524590163934, |
| "grad_norm": 55.672119140625, |
| "learning_rate": 2.819805953336597e-06, |
| "logp/chosen": -816.0, |
| "logp/rejected": -1096.0, |
| "loss": 0.823486328125, |
| "nll_loss": 0.5, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.19921875, |
| "reward/margin": 2.328125, |
| "reward/rejected": -2.125, |
| "step": 665 |
| }, |
| { |
| "approx. KL/chosen": 11.25, |
| "approx. KL/rejected": 376.0, |
| "epoch": 0.6823770491803278, |
| "grad_norm": 25.44585418701172, |
| "learning_rate": 2.8082421011542127e-06, |
| "logp/chosen": -824.0, |
| "logp/rejected": -1160.0, |
| "loss": 0.719970703125, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.451171875, |
| "reward/margin": 3.84375, |
| "reward/rejected": -4.3125, |
| "step": 666 |
| }, |
| { |
| "approx. KL/chosen": 121.0, |
| "approx. KL/rejected": 62.0, |
| "epoch": 0.6834016393442623, |
| "grad_norm": 142.7180633544922, |
| "learning_rate": 2.7966986281780403e-06, |
| "logp/chosen": -1304.0, |
| "logp/rejected": -952.0, |
| "loss": 2.45849609375, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 0.099609375, |
| "reward/rejected": -1.0, |
| "step": 667 |
| }, |
| { |
| "approx. KL/chosen": 50.0, |
| "approx. KL/rejected": 220.0, |
| "epoch": 0.6844262295081968, |
| "grad_norm": 62.853111267089844, |
| "learning_rate": 2.7851756821986573e-06, |
| "logp/chosen": -1004.0, |
| "logp/rejected": -868.0, |
| "loss": 0.884765625, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.40234375, |
| "reward/margin": 3.3125, |
| "reward/rejected": -3.71875, |
| "step": 668 |
| }, |
| { |
| "approx. KL/chosen": 58.0, |
| "approx. KL/rejected": 114.0, |
| "epoch": 0.6854508196721312, |
| "grad_norm": 122.56519317626953, |
| "learning_rate": 2.7736734107438323e-06, |
| "logp/chosen": -956.0, |
| "logp/rejected": -1048.0, |
| "loss": 2.4052734375, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.203125, |
| "reward/margin": 0.00244140625, |
| "reward/rejected": -1.203125, |
| "step": 669 |
| }, |
| { |
| "approx. KL/chosen": 32.25, |
| "approx. KL/rejected": 124.0, |
| "epoch": 0.6864754098360656, |
| "grad_norm": 38.9806022644043, |
| "learning_rate": 2.7621919610766425e-06, |
| "logp/chosen": -708.0, |
| "logp/rejected": -964.0, |
| "loss": 0.8974609375, |
| "nll_loss": 0.490234375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.04443359375, |
| "reward/margin": 2.171875, |
| "reward/rejected": -2.21875, |
| "step": 670 |
| }, |
| { |
| "approx. KL/chosen": 83.0, |
| "approx. KL/rejected": 109.0, |
| "epoch": 0.6875, |
| "grad_norm": 115.29985046386719, |
| "learning_rate": 2.750731480193577e-06, |
| "logp/chosen": -948.0, |
| "logp/rejected": -1096.0, |
| "loss": 2.1904296875, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.3515625, |
| "reward/margin": 0.77734375, |
| "reward/rejected": -2.125, |
| "step": 671 |
| }, |
| { |
| "approx. KL/chosen": 71.0, |
| "approx. KL/rejected": 266.0, |
| "epoch": 0.6885245901639344, |
| "grad_norm": 98.4539794921875, |
| "learning_rate": 2.739292114822672e-06, |
| "logp/chosen": -676.0, |
| "logp/rejected": -1056.0, |
| "loss": 1.4765625, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.3515625, |
| "reward/margin": 1.6953125, |
| "reward/rejected": -3.046875, |
| "step": 672 |
| }, |
| { |
| "approx. KL/chosen": 21.5, |
| "approx. KL/rejected": 284.0, |
| "epoch": 0.6895491803278688, |
| "grad_norm": 81.80807495117188, |
| "learning_rate": 2.7278740114216145e-06, |
| "logp/chosen": -676.0, |
| "logp/rejected": -968.0, |
| "loss": 1.2919921875, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.52734375, |
| "reward/margin": 3.25, |
| "reward/rejected": -3.765625, |
| "step": 673 |
| }, |
| { |
| "approx. KL/chosen": 52.25, |
| "approx. KL/rejected": 138.0, |
| "epoch": 0.6905737704918032, |
| "grad_norm": 43.59285354614258, |
| "learning_rate": 2.716477316175883e-06, |
| "logp/chosen": -1240.0, |
| "logp/rejected": -1736.0, |
| "loss": 0.8505859375, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.8515625, |
| "reward/margin": 3.0, |
| "reward/rejected": -2.15625, |
| "step": 674 |
| }, |
| { |
| "approx. KL/chosen": 134.0, |
| "approx. KL/rejected": 95.0, |
| "epoch": 0.6915983606557377, |
| "grad_norm": 95.87407684326172, |
| "learning_rate": 2.7051021749968606e-06, |
| "logp/chosen": -872.0, |
| "logp/rejected": -940.0, |
| "loss": 2.1044921875, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.298828125, |
| "reward/margin": 1.6015625, |
| "reward/rejected": -1.90625, |
| "step": 675 |
| }, |
| { |
| "approx. KL/chosen": 26.5, |
| "approx. KL/rejected": 99.0, |
| "epoch": 0.6926229508196722, |
| "grad_norm": 50.512638092041016, |
| "learning_rate": 2.693748733519981e-06, |
| "logp/chosen": -736.0, |
| "logp/rejected": -844.0, |
| "loss": 1.00341796875, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.1748046875, |
| "reward/margin": 2.046875, |
| "reward/rejected": -2.21875, |
| "step": 676 |
| }, |
| { |
| "approx. KL/chosen": 46.75, |
| "approx. KL/rejected": 656.0, |
| "epoch": 0.6936475409836066, |
| "grad_norm": 120.7043685913086, |
| "learning_rate": 2.6824171371028507e-06, |
| "logp/chosen": -744.0, |
| "logp/rejected": -1624.0, |
| "loss": 1.58740234375, |
| "nll_loss": 0.44921875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.140625, |
| "reward/margin": 2.484375, |
| "reward/rejected": -3.625, |
| "step": 677 |
| }, |
| { |
| "approx. KL/chosen": 21.25, |
| "approx. KL/rejected": 268.0, |
| "epoch": 0.694672131147541, |
| "grad_norm": 53.649192810058594, |
| "learning_rate": 2.6711075308234055e-06, |
| "logp/chosen": -1136.0, |
| "logp/rejected": -1240.0, |
| "loss": 0.8154296875, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.75, |
| "reward/margin": 2.796875, |
| "reward/rejected": -3.5625, |
| "step": 678 |
| }, |
| { |
| "approx. KL/chosen": 18.0, |
| "approx. KL/rejected": 148.0, |
| "epoch": 0.6956967213114754, |
| "grad_norm": 86.42240905761719, |
| "learning_rate": 2.659820059478031e-06, |
| "logp/chosen": -1020.0, |
| "logp/rejected": -1544.0, |
| "loss": 1.07763671875, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.57421875, |
| "reward/margin": 1.984375, |
| "reward/rejected": -2.5625, |
| "step": 679 |
| }, |
| { |
| "approx. KL/chosen": 101.0, |
| "approx. KL/rejected": 312.0, |
| "epoch": 0.6967213114754098, |
| "grad_norm": 102.62972259521484, |
| "learning_rate": 2.6485548675797286e-06, |
| "logp/chosen": -780.0, |
| "logp/rejected": -1176.0, |
| "loss": 1.73876953125, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.5, |
| "reward/margin": 2.359375, |
| "reward/rejected": -3.84375, |
| "step": 680 |
| }, |
| { |
| "approx. KL/chosen": 57.5, |
| "approx. KL/rejected": 244.0, |
| "epoch": 0.6977459016393442, |
| "grad_norm": 107.11853790283203, |
| "learning_rate": 2.6373120993562494e-06, |
| "logp/chosen": -940.0, |
| "logp/rejected": -1600.0, |
| "loss": 1.57080078125, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.5, |
| "reward/margin": 2.453125, |
| "reward/rejected": -2.953125, |
| "step": 681 |
| }, |
| { |
| "approx. KL/chosen": 141.0, |
| "approx. KL/rejected": 220.0, |
| "epoch": 0.6987704918032787, |
| "grad_norm": 128.02220153808594, |
| "learning_rate": 2.626091898748265e-06, |
| "logp/chosen": -1648.0, |
| "logp/rejected": -2256.0, |
| "loss": 1.955078125, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.5, |
| "reward/margin": 1.5078125, |
| "reward/rejected": -3.0, |
| "step": 682 |
| }, |
| { |
| "approx. KL/chosen": 122.0, |
| "approx. KL/rejected": 146.0, |
| "epoch": 0.6997950819672131, |
| "grad_norm": 150.161376953125, |
| "learning_rate": 2.614894409407504e-06, |
| "logp/chosen": -1312.0, |
| "logp/rejected": -1088.0, |
| "loss": 2.5185546875, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -1.8046875, |
| "reward/margin": -0.05078125, |
| "reward/rejected": -1.75, |
| "step": 683 |
| }, |
| { |
| "approx. KL/chosen": 18.5, |
| "approx. KL/rejected": 163.0, |
| "epoch": 0.7008196721311475, |
| "grad_norm": 21.250699996948242, |
| "learning_rate": 2.603719774694933e-06, |
| "logp/chosen": -808.0, |
| "logp/rejected": -880.0, |
| "loss": 0.7177734375, |
| "nll_loss": 0.52734375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.07568359375, |
| "reward/margin": 2.578125, |
| "reward/rejected": -2.65625, |
| "step": 684 |
| }, |
| { |
| "approx. KL/chosen": 152.0, |
| "approx. KL/rejected": 89.0, |
| "epoch": 0.701844262295082, |
| "grad_norm": 150.5844268798828, |
| "learning_rate": 2.5925681376789044e-06, |
| "logp/chosen": -1280.0, |
| "logp/rejected": -1280.0, |
| "loss": 2.4560546875, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.6015625, |
| "reward/margin": 0.1025390625, |
| "reward/rejected": -1.703125, |
| "step": 685 |
| }, |
| { |
| "approx. KL/chosen": 76.0, |
| "approx. KL/rejected": 344.0, |
| "epoch": 0.7028688524590164, |
| "grad_norm": 90.87260437011719, |
| "learning_rate": 2.5814396411333354e-06, |
| "logp/chosen": -920.0, |
| "logp/rejected": -1688.0, |
| "loss": 1.619140625, |
| "nll_loss": 0.5078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0, |
| "reward/margin": 2.25, |
| "reward/rejected": -3.25, |
| "step": 686 |
| }, |
| { |
| "approx. KL/chosen": 20.25, |
| "approx. KL/rejected": 123.0, |
| "epoch": 0.7038934426229508, |
| "grad_norm": 48.61743927001953, |
| "learning_rate": 2.5703344275358803e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -924.0, |
| "loss": 0.739013671875, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.65234375, |
| "reward/margin": 3.078125, |
| "reward/rejected": -2.421875, |
| "step": 687 |
| }, |
| { |
| "approx. KL/chosen": 87.5, |
| "approx. KL/rejected": 110.5, |
| "epoch": 0.7049180327868853, |
| "grad_norm": 119.87178039550781, |
| "learning_rate": 2.5592526390660957e-06, |
| "logp/chosen": -900.0, |
| "logp/rejected": -1072.0, |
| "loss": 1.9228515625, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.5234375, |
| "reward/margin": 0.46484375, |
| "reward/rejected": -1.9921875, |
| "step": 688 |
| }, |
| { |
| "approx. KL/chosen": 6.75, |
| "approx. KL/rejected": 194.0, |
| "epoch": 0.7059426229508197, |
| "grad_norm": 28.271718978881836, |
| "learning_rate": 2.5481944176036338e-06, |
| "logp/chosen": -688.0, |
| "logp/rejected": -1004.0, |
| "loss": 0.9306640625, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 2.5625, |
| "reward/rejected": -2.703125, |
| "step": 689 |
| }, |
| { |
| "approx. KL/chosen": 25.0, |
| "approx. KL/rejected": 202.0, |
| "epoch": 0.7069672131147541, |
| "grad_norm": 62.063568115234375, |
| "learning_rate": 2.537159904726412e-06, |
| "logp/chosen": -624.0, |
| "logp/rejected": -968.0, |
| "loss": 0.91748046875, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.275390625, |
| "reward/margin": 2.78125, |
| "reward/rejected": -3.0625, |
| "step": 690 |
| }, |
| { |
| "approx. KL/chosen": 10.8125, |
| "approx. KL/rejected": 119.0, |
| "epoch": 0.7079918032786885, |
| "grad_norm": 61.80892562866211, |
| "learning_rate": 2.5261492417088176e-06, |
| "logp/chosen": -756.0, |
| "logp/rejected": -952.0, |
| "loss": 0.93994140625, |
| "nll_loss": 0.6484375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.2255859375, |
| "reward/margin": 2.28125, |
| "reward/rejected": -2.5, |
| "step": 691 |
| }, |
| { |
| "approx. KL/chosen": 17.25, |
| "approx. KL/rejected": 198.0, |
| "epoch": 0.7090163934426229, |
| "grad_norm": 85.72180938720703, |
| "learning_rate": 2.51516256951988e-06, |
| "logp/chosen": -944.0, |
| "logp/rejected": -1448.0, |
| "loss": 0.919189453125, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.94921875, |
| "reward/margin": 2.28125, |
| "reward/rejected": -3.21875, |
| "step": 692 |
| }, |
| { |
| "approx. KL/chosen": 72.5, |
| "approx. KL/rejected": 342.0, |
| "epoch": 0.7100409836065574, |
| "grad_norm": 57.5715217590332, |
| "learning_rate": 2.504200028821482e-06, |
| "logp/chosen": -1000.0, |
| "logp/rejected": -1320.0, |
| "loss": 1.77685546875, |
| "nll_loss": 0.6640625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 2.6875, |
| "reward/rejected": -3.578125, |
| "step": 693 |
| }, |
| { |
| "approx. KL/chosen": 23.25, |
| "approx. KL/rejected": 276.0, |
| "epoch": 0.7110655737704918, |
| "grad_norm": 35.323665618896484, |
| "learning_rate": 2.493261759966545e-06, |
| "logp/chosen": -900.0, |
| "logp/rejected": -1416.0, |
| "loss": 0.8525390625, |
| "nll_loss": 0.6640625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.55078125, |
| "reward/margin": 3.5625, |
| "reward/rejected": -3.0, |
| "step": 694 |
| }, |
| { |
| "approx. KL/chosen": 58.25, |
| "approx. KL/rejected": 198.0, |
| "epoch": 0.7120901639344263, |
| "grad_norm": 99.37004852294922, |
| "learning_rate": 2.4823479029972473e-06, |
| "logp/chosen": -1080.0, |
| "logp/rejected": -1304.0, |
| "loss": 1.2138671875, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.25, |
| "reward/margin": 1.7578125, |
| "reward/rejected": -3.0, |
| "step": 695 |
| }, |
| { |
| "approx. KL/chosen": 12.25, |
| "approx. KL/rejected": 278.0, |
| "epoch": 0.7131147540983607, |
| "grad_norm": 74.05281829833984, |
| "learning_rate": 2.471458597643215e-06, |
| "logp/chosen": -904.0, |
| "logp/rejected": -1216.0, |
| "loss": 1.1962890625, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.4140625, |
| "reward/margin": 2.609375, |
| "reward/rejected": -3.03125, |
| "step": 696 |
| }, |
| { |
| "approx. KL/chosen": 52.0, |
| "approx. KL/rejected": 229.0, |
| "epoch": 0.7141393442622951, |
| "grad_norm": 82.0306396484375, |
| "learning_rate": 2.4605939833197474e-06, |
| "logp/chosen": -996.0, |
| "logp/rejected": -1448.0, |
| "loss": 1.0595703125, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.201171875, |
| "reward/margin": 2.90625, |
| "reward/rejected": -3.109375, |
| "step": 697 |
| }, |
| { |
| "approx. KL/chosen": 22.875, |
| "approx. KL/rejected": 163.0, |
| "epoch": 0.7151639344262295, |
| "grad_norm": 95.63095092773438, |
| "learning_rate": 2.449754199126021e-06, |
| "logp/chosen": -796.0, |
| "logp/rejected": -1168.0, |
| "loss": 1.093017578125, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.36328125, |
| "reward/margin": 2.8125, |
| "reward/rejected": -2.4375, |
| "step": 698 |
| }, |
| { |
| "approx. KL/chosen": 27.75, |
| "approx. KL/rejected": 161.0, |
| "epoch": 0.7161885245901639, |
| "grad_norm": 44.189361572265625, |
| "learning_rate": 2.4389393838433144e-06, |
| "logp/chosen": -716.0, |
| "logp/rejected": -1020.0, |
| "loss": 0.90234375, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.375, |
| "reward/margin": 2.625, |
| "reward/rejected": -3.0, |
| "step": 699 |
| }, |
| { |
| "approx. KL/chosen": 136.0, |
| "approx. KL/rejected": 71.5, |
| "epoch": 0.7172131147540983, |
| "grad_norm": 133.5411834716797, |
| "learning_rate": 2.428149675933233e-06, |
| "logp/chosen": -1248.0, |
| "logp/rejected": -1032.0, |
| "loss": 2.10693359375, |
| "nll_loss": 0.49609375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.86328125, |
| "reward/margin": 0.5390625, |
| "reward/rejected": -1.3984375, |
| "step": 700 |
| }, |
| { |
| "approx. KL/chosen": 198.0, |
| "approx. KL/rejected": 260.0, |
| "epoch": 0.7182377049180327, |
| "grad_norm": 114.77182006835938, |
| "learning_rate": 2.4173852135359304e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1328.0, |
| "loss": 1.956298828125, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -2.40625, |
| "reward/margin": 1.3828125, |
| "reward/rejected": -3.796875, |
| "step": 701 |
| }, |
| { |
| "approx. KL/chosen": 85.5, |
| "approx. KL/rejected": 197.0, |
| "epoch": 0.7192622950819673, |
| "grad_norm": 105.85591125488281, |
| "learning_rate": 2.4066461344683442e-06, |
| "logp/chosen": -932.0, |
| "logp/rejected": -916.0, |
| "loss": 1.44384765625, |
| "nll_loss": 0.515625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.3046875, |
| "reward/margin": 1.53125, |
| "reward/rejected": -2.828125, |
| "step": 702 |
| }, |
| { |
| "approx. KL/chosen": 41.25, |
| "approx. KL/rejected": 278.0, |
| "epoch": 0.7202868852459017, |
| "grad_norm": 70.50735473632812, |
| "learning_rate": 2.3959325762224302e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1752.0, |
| "loss": 0.96435546875, |
| "nll_loss": 0.69140625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.55078125, |
| "reward/margin": 4.21875, |
| "reward/rejected": -3.65625, |
| "step": 703 |
| }, |
| { |
| "approx. KL/chosen": 14.25, |
| "approx. KL/rejected": 218.0, |
| "epoch": 0.7213114754098361, |
| "grad_norm": 34.13417053222656, |
| "learning_rate": 2.3852446759634026e-06, |
| "logp/chosen": -1064.0, |
| "logp/rejected": -1352.0, |
| "loss": 0.732421875, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.050048828125, |
| "reward/margin": 3.796875, |
| "reward/rejected": -3.859375, |
| "step": 704 |
| }, |
| { |
| "approx. KL/chosen": 7.0625, |
| "approx. KL/rejected": 196.0, |
| "epoch": 0.7223360655737705, |
| "grad_norm": 59.88851547241211, |
| "learning_rate": 2.3745825705279787e-06, |
| "logp/chosen": -498.0, |
| "logp/rejected": -784.0, |
| "loss": 0.91015625, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.57421875, |
| "reward/margin": 2.453125, |
| "reward/rejected": -3.03125, |
| "step": 705 |
| }, |
| { |
| "approx. KL/chosen": 135.0, |
| "approx. KL/rejected": 77.0, |
| "epoch": 0.7233606557377049, |
| "grad_norm": 141.93052673339844, |
| "learning_rate": 2.3639463964226213e-06, |
| "logp/chosen": -1072.0, |
| "logp/rejected": -984.0, |
| "loss": 2.246826171875, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.67578125, |
| "reward/margin": 0.4296875, |
| "reward/rejected": -1.109375, |
| "step": 706 |
| }, |
| { |
| "approx. KL/chosen": 31.625, |
| "approx. KL/rejected": 185.0, |
| "epoch": 0.7243852459016393, |
| "grad_norm": 66.92444610595703, |
| "learning_rate": 2.3533362898218e-06, |
| "logp/chosen": -792.0, |
| "logp/rejected": -1208.0, |
| "loss": 0.93701171875, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 2.0625, |
| "reward/rejected": -2.859375, |
| "step": 707 |
| }, |
| { |
| "approx. KL/chosen": 42.25, |
| "approx. KL/rejected": 71.5, |
| "epoch": 0.7254098360655737, |
| "grad_norm": 97.35623168945312, |
| "learning_rate": 2.342752386566243e-06, |
| "logp/chosen": -876.0, |
| "logp/rejected": -1080.0, |
| "loss": 1.80224609375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.5625, |
| "reward/margin": -0.263671875, |
| "reward/rejected": -1.3046875, |
| "step": 708 |
| }, |
| { |
| "approx. KL/chosen": 9.25, |
| "approx. KL/rejected": 336.0, |
| "epoch": 0.7264344262295082, |
| "grad_norm": 22.90494728088379, |
| "learning_rate": 2.332194822161197e-06, |
| "logp/chosen": -768.0, |
| "logp/rejected": -1200.0, |
| "loss": 0.6865234375, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.25, |
| "reward/margin": 4.5, |
| "reward/rejected": -4.25, |
| "step": 709 |
| }, |
| { |
| "approx. KL/chosen": 72.0, |
| "approx. KL/rejected": 278.0, |
| "epoch": 0.7274590163934426, |
| "grad_norm": 161.59109497070312, |
| "learning_rate": 2.3216637317746925e-06, |
| "logp/chosen": -1272.0, |
| "logp/rejected": -1376.0, |
| "loss": 2.0556640625, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.8046875, |
| "reward/margin": 0.62109375, |
| "reward/rejected": -2.421875, |
| "step": 710 |
| }, |
| { |
| "approx. KL/chosen": 38.25, |
| "approx. KL/rejected": 46.25, |
| "epoch": 0.7284836065573771, |
| "grad_norm": 84.78888702392578, |
| "learning_rate": 2.3111592502358154e-06, |
| "logp/chosen": -1376.0, |
| "logp/rejected": -864.0, |
| "loss": 1.673583984375, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.3515625, |
| "reward/margin": 0.375, |
| "reward/rejected": -0.7265625, |
| "step": 711 |
| }, |
| { |
| "approx. KL/chosen": 25.5, |
| "approx. KL/rejected": 199.0, |
| "epoch": 0.7295081967213115, |
| "grad_norm": 68.85263061523438, |
| "learning_rate": 2.3006815120329806e-06, |
| "logp/chosen": -872.0, |
| "logp/rejected": -1232.0, |
| "loss": 0.834716796875, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.10009765625, |
| "reward/margin": 3.53125, |
| "reward/rejected": -3.4375, |
| "step": 712 |
| }, |
| { |
| "approx. KL/chosen": 103.0, |
| "approx. KL/rejected": 142.0, |
| "epoch": 0.7305327868852459, |
| "grad_norm": 79.05963134765625, |
| "learning_rate": 2.29023065131221e-06, |
| "logp/chosen": -800.0, |
| "logp/rejected": -1048.0, |
| "loss": 1.27880859375, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.5, |
| "reward/margin": 1.25, |
| "reward/rejected": -2.75, |
| "step": 713 |
| }, |
| { |
| "approx. KL/chosen": 40.5, |
| "approx. KL/rejected": 102.0, |
| "epoch": 0.7315573770491803, |
| "grad_norm": 58.485591888427734, |
| "learning_rate": 2.279806801875409e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1080.0, |
| "loss": 1.06982421875, |
| "nll_loss": 0.6484375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 1.3359375, |
| "reward/rejected": -2.4375, |
| "step": 714 |
| }, |
| { |
| "approx. KL/chosen": 28.75, |
| "approx. KL/rejected": 42.5, |
| "epoch": 0.7325819672131147, |
| "grad_norm": 59.67937469482422, |
| "learning_rate": 2.269410097178665e-06, |
| "logp/chosen": -712.0, |
| "logp/rejected": -684.0, |
| "loss": 0.8662109375, |
| "nll_loss": 0.47265625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.0255126953125, |
| "reward/margin": 1.3828125, |
| "reward/rejected": -1.40625, |
| "step": 715 |
| }, |
| { |
| "approx. KL/chosen": 73.0, |
| "approx. KL/rejected": 328.0, |
| "epoch": 0.7336065573770492, |
| "grad_norm": 117.98296356201172, |
| "learning_rate": 2.2590406703305294e-06, |
| "logp/chosen": -1184.0, |
| "logp/rejected": -1712.0, |
| "loss": 1.79833984375, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.5, |
| "reward/margin": 2.296875, |
| "reward/rejected": -2.8125, |
| "step": 716 |
| }, |
| { |
| "approx. KL/chosen": 20.0, |
| "approx. KL/rejected": 109.0, |
| "epoch": 0.7346311475409836, |
| "grad_norm": 58.30084228515625, |
| "learning_rate": 2.248698654090319e-06, |
| "logp/chosen": -912.0, |
| "logp/rejected": -936.0, |
| "loss": 0.958251953125, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.201171875, |
| "reward/margin": 2.296875, |
| "reward/rejected": -2.109375, |
| "step": 717 |
| }, |
| { |
| "approx. KL/chosen": 52.25, |
| "approx. KL/rejected": 198.0, |
| "epoch": 0.735655737704918, |
| "grad_norm": 27.151552200317383, |
| "learning_rate": 2.238384180866408e-06, |
| "logp/chosen": -840.0, |
| "logp/rejected": -1304.0, |
| "loss": 0.82666015625, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.296875, |
| "reward/margin": 3.078125, |
| "reward/rejected": -2.78125, |
| "step": 718 |
| }, |
| { |
| "approx. KL/chosen": 87.0, |
| "approx. KL/rejected": 264.0, |
| "epoch": 0.7366803278688525, |
| "grad_norm": 102.41096496582031, |
| "learning_rate": 2.2280973827145434e-06, |
| "logp/chosen": -896.0, |
| "logp/rejected": -1056.0, |
| "loss": 1.413330078125, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.48828125, |
| "reward/margin": 3.015625, |
| "reward/rejected": -3.5, |
| "step": 719 |
| }, |
| { |
| "approx. KL/chosen": 64.5, |
| "approx. KL/rejected": 70.0, |
| "epoch": 0.7377049180327869, |
| "grad_norm": 121.46066284179688, |
| "learning_rate": 2.2178383913361503e-06, |
| "logp/chosen": -916.0, |
| "logp/rejected": -1004.0, |
| "loss": 2.18115234375, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.0, |
| "reward/margin": 0.1767578125, |
| "reward/rejected": -1.1796875, |
| "step": 720 |
| }, |
| { |
| "approx. KL/chosen": 10.0, |
| "approx. KL/rejected": 448.0, |
| "epoch": 0.7387295081967213, |
| "grad_norm": 76.36231994628906, |
| "learning_rate": 2.207607338076644e-06, |
| "logp/chosen": -1280.0, |
| "logp/rejected": -1344.0, |
| "loss": 1.10693359375, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.0, |
| "reward/margin": 3.90625, |
| "reward/rejected": -3.90625, |
| "step": 721 |
| }, |
| { |
| "approx. KL/chosen": 31.5, |
| "approx. KL/rejected": 356.0, |
| "epoch": 0.7397540983606558, |
| "grad_norm": 6.144227504730225, |
| "learning_rate": 2.197404353923745e-06, |
| "logp/chosen": -944.0, |
| "logp/rejected": -1656.0, |
| "loss": 0.73046875, |
| "nll_loss": 0.70703125, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.025390625, |
| "reward/margin": 5.03125, |
| "reward/rejected": -5.0, |
| "step": 722 |
| }, |
| { |
| "approx. KL/chosen": 15.8125, |
| "approx. KL/rejected": 82.5, |
| "epoch": 0.7407786885245902, |
| "grad_norm": 81.19537353515625, |
| "learning_rate": 2.1872295695058125e-06, |
| "logp/chosen": -1000.0, |
| "logp/rejected": -1136.0, |
| "loss": 1.36962890625, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.578125, |
| "reward/margin": 1.03125, |
| "reward/rejected": -1.6015625, |
| "step": 723 |
| }, |
| { |
| "approx. KL/chosen": 43.5, |
| "approx. KL/rejected": 53.5, |
| "epoch": 0.7418032786885246, |
| "grad_norm": 110.544677734375, |
| "learning_rate": 2.1770831150901583e-06, |
| "logp/chosen": -724.0, |
| "logp/rejected": -828.0, |
| "loss": 2.0341796875, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.203125, |
| "reward/margin": -0.400390625, |
| "reward/rejected": -0.80078125, |
| "step": 724 |
| }, |
| { |
| "approx. KL/chosen": 15.0, |
| "approx. KL/rejected": 125.0, |
| "epoch": 0.742827868852459, |
| "grad_norm": 28.955930709838867, |
| "learning_rate": 2.166965120581398e-06, |
| "logp/chosen": -1020.0, |
| "logp/rejected": -1328.0, |
| "loss": 0.68310546875, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.10009765625, |
| "reward/margin": 2.796875, |
| "reward/rejected": -2.703125, |
| "step": 725 |
| }, |
| { |
| "approx. KL/chosen": 75.5, |
| "approx. KL/rejected": 198.0, |
| "epoch": 0.7438524590163934, |
| "grad_norm": 47.21245193481445, |
| "learning_rate": 2.156875715519764e-06, |
| "logp/chosen": -880.0, |
| "logp/rejected": -1128.0, |
| "loss": 0.8271484375, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 1.0, |
| "reward/margin": 4.25, |
| "reward/rejected": -3.25, |
| "step": 726 |
| }, |
| { |
| "approx. KL/chosen": 36.75, |
| "approx. KL/rejected": 143.0, |
| "epoch": 0.7448770491803278, |
| "grad_norm": 74.15019989013672, |
| "learning_rate": 2.146815029079468e-06, |
| "logp/chosen": -540.0, |
| "logp/rejected": -812.0, |
| "loss": 1.328125, |
| "nll_loss": 0.43359375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.125, |
| "reward/margin": 1.2265625, |
| "reward/rejected": -2.359375, |
| "step": 727 |
| }, |
| { |
| "approx. KL/chosen": 31.0, |
| "approx. KL/rejected": 174.0, |
| "epoch": 0.7459016393442623, |
| "grad_norm": 55.888221740722656, |
| "learning_rate": 2.1367831900670298e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -1012.0, |
| "loss": 0.907470703125, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.30078125, |
| "reward/margin": 2.515625, |
| "reward/rejected": -2.21875, |
| "step": 728 |
| }, |
| { |
| "approx. KL/chosen": 75.0, |
| "approx. KL/rejected": 97.0, |
| "epoch": 0.7469262295081968, |
| "grad_norm": 112.5295181274414, |
| "learning_rate": 2.1267803269196488e-06, |
| "logp/chosen": -1144.0, |
| "logp/rejected": -1208.0, |
| "loss": 1.36083984375, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.94921875, |
| "reward/margin": 1.1171875, |
| "reward/rejected": -2.0625, |
| "step": 729 |
| }, |
| { |
| "approx. KL/chosen": 20.75, |
| "approx. KL/rejected": 540.0, |
| "epoch": 0.7479508196721312, |
| "grad_norm": 42.13194274902344, |
| "learning_rate": 2.1168065677035363e-06, |
| "logp/chosen": -1112.0, |
| "logp/rejected": -1728.0, |
| "loss": 0.85888671875, |
| "nll_loss": 0.6484375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.57421875, |
| "reward/margin": 5.4375, |
| "reward/rejected": -4.84375, |
| "step": 730 |
| }, |
| { |
| "approx. KL/chosen": 181.0, |
| "approx. KL/rejected": 217.0, |
| "epoch": 0.7489754098360656, |
| "grad_norm": 115.49468231201172, |
| "learning_rate": 2.1068620401122937e-06, |
| "logp/chosen": -1672.0, |
| "logp/rejected": -1328.0, |
| "loss": 1.880615234375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.875, |
| "reward/margin": 1.7734375, |
| "reward/rejected": -3.65625, |
| "step": 731 |
| }, |
| { |
| "approx. KL/chosen": 15.0625, |
| "approx. KL/rejected": 296.0, |
| "epoch": 0.75, |
| "grad_norm": 2.0833182334899902, |
| "learning_rate": 2.0969468714652672e-06, |
| "logp/chosen": -756.0, |
| "logp/rejected": -1248.0, |
| "loss": 0.634765625, |
| "nll_loss": 0.625, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.4765625, |
| "reward/margin": 5.15625, |
| "reward/rejected": -4.6875, |
| "step": 732 |
| }, |
| { |
| "approx. KL/chosen": 83.0, |
| "approx. KL/rejected": 62.0, |
| "epoch": 0.7510245901639344, |
| "grad_norm": 101.0868911743164, |
| "learning_rate": 2.0870611887059284e-06, |
| "logp/chosen": -760.0, |
| "logp/rejected": -736.0, |
| "loss": 2.0400390625, |
| "nll_loss": 0.5078125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.828125, |
| "reward/margin": -0.546875, |
| "reward/rejected": -1.28125, |
| "step": 733 |
| }, |
| { |
| "approx. KL/chosen": 104.0, |
| "approx. KL/rejected": 104.0, |
| "epoch": 0.7520491803278688, |
| "grad_norm": 142.1163330078125, |
| "learning_rate": 2.0772051184002327e-06, |
| "logp/chosen": -1560.0, |
| "logp/rejected": -1584.0, |
| "loss": 1.611328125, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 0.80078125, |
| "reward/rejected": -1.6015625, |
| "step": 734 |
| }, |
| { |
| "approx. KL/chosen": 27.25, |
| "approx. KL/rejected": 450.0, |
| "epoch": 0.7530737704918032, |
| "grad_norm": 10.717184066772461, |
| "learning_rate": 2.067378786735018e-06, |
| "logp/chosen": -924.0, |
| "logp/rejected": -1408.0, |
| "loss": 0.666015625, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.150390625, |
| "reward/margin": 5.5625, |
| "reward/rejected": -5.40625, |
| "step": 735 |
| }, |
| { |
| "approx. KL/chosen": 32.5, |
| "approx. KL/rejected": 172.0, |
| "epoch": 0.7540983606557377, |
| "grad_norm": 88.11271667480469, |
| "learning_rate": 2.0575823195163737e-06, |
| "logp/chosen": -984.0, |
| "logp/rejected": -1240.0, |
| "loss": 1.4541015625, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 2.046875, |
| "reward/rejected": -2.453125, |
| "step": 736 |
| }, |
| { |
| "approx. KL/chosen": 44.5, |
| "approx. KL/rejected": 49.25, |
| "epoch": 0.7551229508196722, |
| "grad_norm": 98.76152038574219, |
| "learning_rate": 2.0478158421680354e-06, |
| "logp/chosen": -784.0, |
| "logp/rejected": -544.0, |
| "loss": 1.528564453125, |
| "nll_loss": 0.47265625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 0.4140625, |
| "reward/rejected": -1.2109375, |
| "step": 737 |
| }, |
| { |
| "approx. KL/chosen": 71.0, |
| "approx. KL/rejected": 408.0, |
| "epoch": 0.7561475409836066, |
| "grad_norm": 40.50996780395508, |
| "learning_rate": 2.0380794797297873e-06, |
| "logp/chosen": -1064.0, |
| "logp/rejected": -1408.0, |
| "loss": 0.765625, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -1.0546875, |
| "reward/margin": 3.65625, |
| "reward/rejected": -4.71875, |
| "step": 738 |
| }, |
| { |
| "approx. KL/chosen": 29.0, |
| "approx. KL/rejected": 376.0, |
| "epoch": 0.757172131147541, |
| "grad_norm": 96.07220458984375, |
| "learning_rate": 2.0283733568558445e-06, |
| "logp/chosen": -1656.0, |
| "logp/rejected": -1552.0, |
| "loss": 1.033203125, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 4.65625, |
| "reward/rejected": -4.78125, |
| "step": 739 |
| }, |
| { |
| "approx. KL/chosen": 37.75, |
| "approx. KL/rejected": 238.0, |
| "epoch": 0.7581967213114754, |
| "grad_norm": 72.74873352050781, |
| "learning_rate": 2.0186975978132722e-06, |
| "logp/chosen": -956.0, |
| "logp/rejected": -1088.0, |
| "loss": 1.22119140625, |
| "nll_loss": 0.52734375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.625, |
| "reward/margin": 2.796875, |
| "reward/rejected": -3.421875, |
| "step": 740 |
| }, |
| { |
| "approx. KL/chosen": 53.75, |
| "approx. KL/rejected": 130.0, |
| "epoch": 0.7592213114754098, |
| "grad_norm": 103.7167739868164, |
| "learning_rate": 2.0090523264803846e-06, |
| "logp/chosen": -788.0, |
| "logp/rejected": -1096.0, |
| "loss": 1.513671875, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.078125, |
| "reward/margin": 1.328125, |
| "reward/rejected": -2.40625, |
| "step": 741 |
| }, |
| { |
| "approx. KL/chosen": 21.0, |
| "approx. KL/rejected": 340.0, |
| "epoch": 0.7602459016393442, |
| "grad_norm": 14.779808044433594, |
| "learning_rate": 1.9994376663451684e-06, |
| "logp/chosen": -1096.0, |
| "logp/rejected": -1760.0, |
| "loss": 0.681640625, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.69921875, |
| "reward/margin": 5.34375, |
| "reward/rejected": -4.65625, |
| "step": 742 |
| }, |
| { |
| "approx. KL/chosen": 46.25, |
| "approx. KL/rejected": 146.0, |
| "epoch": 0.7612704918032787, |
| "grad_norm": 79.98866271972656, |
| "learning_rate": 1.9898537405036916e-06, |
| "logp/chosen": -1072.0, |
| "logp/rejected": -1384.0, |
| "loss": 0.9287109375, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.050048828125, |
| "reward/margin": 2.75, |
| "reward/rejected": -2.796875, |
| "step": 743 |
| }, |
| { |
| "approx. KL/chosen": 57.25, |
| "approx. KL/rejected": 82.0, |
| "epoch": 0.7622950819672131, |
| "grad_norm": 131.90664672851562, |
| "learning_rate": 1.9803006716585354e-06, |
| "logp/chosen": -1368.0, |
| "logp/rejected": -1592.0, |
| "loss": 1.80419921875, |
| "nll_loss": 0.515625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.328125, |
| "reward/margin": 0.2734375, |
| "reward/rejected": -1.6015625, |
| "step": 744 |
| }, |
| { |
| "approx. KL/chosen": 86.5, |
| "approx. KL/rejected": 104.0, |
| "epoch": 0.7633196721311475, |
| "grad_norm": 128.1505584716797, |
| "learning_rate": 1.9707785821172174e-06, |
| "logp/chosen": -1192.0, |
| "logp/rejected": -1000.0, |
| "loss": 1.60302734375, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.5, |
| "reward/margin": 0.44921875, |
| "reward/rejected": -1.953125, |
| "step": 745 |
| }, |
| { |
| "approx. KL/chosen": 119.5, |
| "approx. KL/rejected": 143.0, |
| "epoch": 0.764344262295082, |
| "grad_norm": 92.42011260986328, |
| "learning_rate": 1.9612875937906315e-06, |
| "logp/chosen": -1336.0, |
| "logp/rejected": -588.0, |
| "loss": 1.47705078125, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.326171875, |
| "reward/margin": 2.3125, |
| "reward/rejected": -2.640625, |
| "step": 746 |
| }, |
| { |
| "approx. KL/chosen": 19.25, |
| "approx. KL/rejected": 616.0, |
| "epoch": 0.7653688524590164, |
| "grad_norm": 71.03607940673828, |
| "learning_rate": 1.9518278281914826e-06, |
| "logp/chosen": -832.0, |
| "logp/rejected": -1160.0, |
| "loss": 1.369140625, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.349609375, |
| "reward/margin": 4.03125, |
| "reward/rejected": -4.375, |
| "step": 747 |
| }, |
| { |
| "approx. KL/chosen": 54.0, |
| "approx. KL/rejected": 178.0, |
| "epoch": 0.7663934426229508, |
| "grad_norm": 90.6854019165039, |
| "learning_rate": 1.9423994064327343e-06, |
| "logp/chosen": -1000.0, |
| "logp/rejected": -1304.0, |
| "loss": 1.148193359375, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.875, |
| "reward/margin": 2.09375, |
| "reward/rejected": -2.96875, |
| "step": 748 |
| }, |
| { |
| "approx. KL/chosen": 31.0, |
| "approx. KL/rejected": 228.0, |
| "epoch": 0.7674180327868853, |
| "grad_norm": 105.9591064453125, |
| "learning_rate": 1.9330024492260518e-06, |
| "logp/chosen": -1112.0, |
| "logp/rejected": -1408.0, |
| "loss": 1.599609375, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 1.984375, |
| "reward/rejected": -2.890625, |
| "step": 749 |
| }, |
| { |
| "approx. KL/chosen": 28.5, |
| "approx. KL/rejected": 176.0, |
| "epoch": 0.7684426229508197, |
| "grad_norm": 87.54653930664062, |
| "learning_rate": 1.9236370768802658e-06, |
| "logp/chosen": -856.0, |
| "logp/rejected": -1216.0, |
| "loss": 1.12353515625, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 1.40625, |
| "reward/rejected": -2.3125, |
| "step": 750 |
| }, |
| { |
| "approx. KL/chosen": 41.0, |
| "approx. KL/rejected": 61.5, |
| "epoch": 0.7694672131147541, |
| "grad_norm": 106.61505889892578, |
| "learning_rate": 1.9143034092998247e-06, |
| "logp/chosen": -760.0, |
| "logp/rejected": -692.0, |
| "loss": 1.81396484375, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.3984375, |
| "reward/margin": 0.10205078125, |
| "reward/rejected": -1.5, |
| "step": 751 |
| }, |
| { |
| "approx. KL/chosen": 36.5, |
| "approx. KL/rejected": 67.5, |
| "epoch": 0.7704918032786885, |
| "grad_norm": 78.41443634033203, |
| "learning_rate": 1.9050015659832653e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -1000.0, |
| "loss": 1.3740234375, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 1.3984375, |
| "reward/rejected": -1.703125, |
| "step": 752 |
| }, |
| { |
| "approx. KL/chosen": 30.0, |
| "approx. KL/rejected": 233.0, |
| "epoch": 0.7715163934426229, |
| "grad_norm": 44.70865249633789, |
| "learning_rate": 1.8957316660216743e-06, |
| "logp/chosen": -1144.0, |
| "logp/rejected": -1608.0, |
| "loss": 0.73095703125, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.0, |
| "reward/margin": 4.1875, |
| "reward/rejected": -4.1875, |
| "step": 753 |
| }, |
| { |
| "approx. KL/chosen": 23.25, |
| "approx. KL/rejected": 189.0, |
| "epoch": 0.7725409836065574, |
| "grad_norm": 37.49919128417969, |
| "learning_rate": 1.8864938280971743e-06, |
| "logp/chosen": -728.0, |
| "logp/rejected": -960.0, |
| "loss": 0.966796875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 2.703125, |
| "reward/rejected": -3.25, |
| "step": 754 |
| }, |
| { |
| "approx. KL/chosen": 140.0, |
| "approx. KL/rejected": 186.0, |
| "epoch": 0.7735655737704918, |
| "grad_norm": 134.61546325683594, |
| "learning_rate": 1.8772881704813964e-06, |
| "logp/chosen": -976.0, |
| "logp/rejected": -1440.0, |
| "loss": 2.37890625, |
| "nll_loss": 0.52734375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -2.15625, |
| "reward/margin": 0.224609375, |
| "reward/rejected": -2.375, |
| "step": 755 |
| }, |
| { |
| "approx. KL/chosen": 35.25, |
| "approx. KL/rejected": 131.0, |
| "epoch": 0.7745901639344263, |
| "grad_norm": 99.45195007324219, |
| "learning_rate": 1.868114811033971e-06, |
| "logp/chosen": -1152.0, |
| "logp/rejected": -1112.0, |
| "loss": 1.283203125, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.75, |
| "reward/margin": 1.7265625, |
| "reward/rejected": -2.46875, |
| "step": 756 |
| }, |
| { |
| "approx. KL/chosen": 4.125, |
| "approx. KL/rejected": 225.0, |
| "epoch": 0.7756147540983607, |
| "grad_norm": 49.0865592956543, |
| "learning_rate": 1.8589738672010118e-06, |
| "logp/chosen": -592.0, |
| "logp/rejected": -852.0, |
| "loss": 0.882080078125, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.25, |
| "reward/margin": 3.0625, |
| "reward/rejected": -3.3125, |
| "step": 757 |
| }, |
| { |
| "approx. KL/chosen": 51.5, |
| "approx. KL/rejected": 190.0, |
| "epoch": 0.7766393442622951, |
| "grad_norm": 79.05085754394531, |
| "learning_rate": 1.849865456013621e-06, |
| "logp/chosen": -968.0, |
| "logp/rejected": -924.0, |
| "loss": 1.2940673828125, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.3984375, |
| "reward/margin": 1.765625, |
| "reward/rejected": -3.171875, |
| "step": 758 |
| }, |
| { |
| "approx. KL/chosen": 58.0, |
| "approx. KL/rejected": 332.0, |
| "epoch": 0.7776639344262295, |
| "grad_norm": 71.33379364013672, |
| "learning_rate": 1.8407896940863863e-06, |
| "logp/chosen": -1096.0, |
| "logp/rejected": -1320.0, |
| "loss": 1.09375, |
| "nll_loss": 0.52734375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.19921875, |
| "reward/margin": 2.875, |
| "reward/rejected": -2.671875, |
| "step": 759 |
| }, |
| { |
| "approx. KL/chosen": 231.0, |
| "approx. KL/rejected": 120.0, |
| "epoch": 0.7786885245901639, |
| "grad_norm": 179.02879333496094, |
| "learning_rate": 1.8317466976158877e-06, |
| "logp/chosen": -1352.0, |
| "logp/rejected": -1312.0, |
| "loss": 2.769775390625, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -2.4375, |
| "reward/margin": -0.15234375, |
| "reward/rejected": -2.28125, |
| "step": 760 |
| }, |
| { |
| "approx. KL/chosen": 67.5, |
| "approx. KL/rejected": 342.0, |
| "epoch": 0.7797131147540983, |
| "grad_norm": 75.92793273925781, |
| "learning_rate": 1.8227365823792087e-06, |
| "logp/chosen": -844.0, |
| "logp/rejected": -1344.0, |
| "loss": 1.087890625, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.71875, |
| "reward/margin": 2.28125, |
| "reward/rejected": -4.0, |
| "step": 761 |
| }, |
| { |
| "approx. KL/chosen": 18.75, |
| "approx. KL/rejected": 266.0, |
| "epoch": 0.7807377049180327, |
| "grad_norm": 52.53029251098633, |
| "learning_rate": 1.8137594637324587e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1296.0, |
| "loss": 0.90673828125, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.52734375, |
| "reward/margin": 3.875, |
| "reward/rejected": -3.34375, |
| "step": 762 |
| }, |
| { |
| "approx. KL/chosen": 49.0, |
| "approx. KL/rejected": 91.5, |
| "epoch": 0.7817622950819673, |
| "grad_norm": 100.81391143798828, |
| "learning_rate": 1.804815456609292e-06, |
| "logp/chosen": -1088.0, |
| "logp/rejected": -1144.0, |
| "loss": 1.51953125, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.52734375, |
| "reward/margin": 1.34375, |
| "reward/rejected": -1.875, |
| "step": 763 |
| }, |
| { |
| "approx. KL/chosen": 76.0, |
| "approx. KL/rejected": 290.0, |
| "epoch": 0.7827868852459017, |
| "grad_norm": 28.659210205078125, |
| "learning_rate": 1.7959046755194378e-06, |
| "logp/chosen": -1160.0, |
| "logp/rejected": -1020.0, |
| "loss": 0.614501953125, |
| "nll_loss": 0.474609375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.44921875, |
| "reward/margin": 4.25, |
| "reward/rejected": -3.8125, |
| "step": 764 |
| }, |
| { |
| "approx. KL/chosen": 99.0, |
| "approx. KL/rejected": 156.0, |
| "epoch": 0.7838114754098361, |
| "grad_norm": 125.01193237304688, |
| "learning_rate": 1.787027234547233e-06, |
| "logp/chosen": -1008.0, |
| "logp/rejected": -1584.0, |
| "loss": 1.6240234375, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.69921875, |
| "reward/margin": 2.046875, |
| "reward/rejected": -2.75, |
| "step": 765 |
| }, |
| { |
| "approx. KL/chosen": 25.25, |
| "approx. KL/rejected": 100.0, |
| "epoch": 0.7848360655737705, |
| "grad_norm": 81.2750473022461, |
| "learning_rate": 1.7781832473501627e-06, |
| "logp/chosen": -840.0, |
| "logp/rejected": -1064.0, |
| "loss": 0.99462890625, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.375, |
| "reward/margin": 2.09375, |
| "reward/rejected": -1.7109375, |
| "step": 766 |
| }, |
| { |
| "approx. KL/chosen": 33.25, |
| "approx. KL/rejected": 270.0, |
| "epoch": 0.7858606557377049, |
| "grad_norm": 43.08438491821289, |
| "learning_rate": 1.7693728271574058e-06, |
| "logp/chosen": -696.0, |
| "logp/rejected": -1232.0, |
| "loss": 0.955078125, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.4765625, |
| "reward/margin": 3.0625, |
| "reward/rejected": -3.53125, |
| "step": 767 |
| }, |
| { |
| "approx. KL/chosen": 35.0, |
| "approx. KL/rejected": 170.0, |
| "epoch": 0.7868852459016393, |
| "grad_norm": 59.612396240234375, |
| "learning_rate": 1.7605960867683844e-06, |
| "logp/chosen": -760.0, |
| "logp/rejected": -1160.0, |
| "loss": 0.8662109375, |
| "nll_loss": 0.515625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.224609375, |
| "reward/margin": 2.921875, |
| "reward/rejected": -2.703125, |
| "step": 768 |
| }, |
| { |
| "approx. KL/chosen": 7.0, |
| "approx. KL/rejected": 118.0, |
| "epoch": 0.7879098360655737, |
| "grad_norm": 95.52963256835938, |
| "learning_rate": 1.7518531385513172e-06, |
| "logp/chosen": -1016.0, |
| "logp/rejected": -1656.0, |
| "loss": 1.27783203125, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.10009765625, |
| "reward/margin": 1.8984375, |
| "reward/rejected": -1.8046875, |
| "step": 769 |
| }, |
| { |
| "approx. KL/chosen": 30.5, |
| "approx. KL/rejected": 173.0, |
| "epoch": 0.7889344262295082, |
| "grad_norm": 74.76082611083984, |
| "learning_rate": 1.7431440944417852e-06, |
| "logp/chosen": -1056.0, |
| "logp/rejected": -1280.0, |
| "loss": 1.0517578125, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 1.1015625, |
| "reward/margin": 3.03125, |
| "reward/rejected": -1.9296875, |
| "step": 770 |
| }, |
| { |
| "approx. KL/chosen": 22.5, |
| "approx. KL/rejected": 214.0, |
| "epoch": 0.7899590163934426, |
| "grad_norm": 66.5205307006836, |
| "learning_rate": 1.734469065941297e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1600.0, |
| "loss": 0.88671875, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.275390625, |
| "reward/margin": 3.421875, |
| "reward/rejected": -3.15625, |
| "step": 771 |
| }, |
| { |
| "approx. KL/chosen": 18.5, |
| "approx. KL/rejected": 276.0, |
| "epoch": 0.7909836065573771, |
| "grad_norm": 64.5533676147461, |
| "learning_rate": 1.7258281641158603e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -904.0, |
| "loss": 1.1220703125, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.326171875, |
| "reward/margin": 2.921875, |
| "reward/rejected": -3.25, |
| "step": 772 |
| }, |
| { |
| "approx. KL/chosen": 82.0, |
| "approx. KL/rejected": 400.0, |
| "epoch": 0.7920081967213115, |
| "grad_norm": 134.40350341796875, |
| "learning_rate": 1.717221499594559e-06, |
| "logp/chosen": -1152.0, |
| "logp/rejected": -1384.0, |
| "loss": 2.48681640625, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.203125, |
| "reward/margin": 0.50390625, |
| "reward/rejected": -1.703125, |
| "step": 773 |
| }, |
| { |
| "approx. KL/chosen": 39.0, |
| "approx. KL/rejected": 141.0, |
| "epoch": 0.7930327868852459, |
| "grad_norm": 94.29887390136719, |
| "learning_rate": 1.7086491825681414e-06, |
| "logp/chosen": -1136.0, |
| "logp/rejected": -1128.0, |
| "loss": 1.19287109375, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.046875, |
| "reward/margin": 1.4921875, |
| "reward/rejected": -2.546875, |
| "step": 774 |
| }, |
| { |
| "approx. KL/chosen": 43.75, |
| "approx. KL/rejected": 45.0, |
| "epoch": 0.7940573770491803, |
| "grad_norm": 104.68294525146484, |
| "learning_rate": 1.700111322787601e-06, |
| "logp/chosen": -740.0, |
| "logp/rejected": -1152.0, |
| "loss": 1.47412109375, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.1796875, |
| "reward/margin": 0.1240234375, |
| "reward/rejected": -1.3046875, |
| "step": 775 |
| }, |
| { |
| "approx. KL/chosen": 59.5, |
| "approx. KL/rejected": 186.0, |
| "epoch": 0.7950819672131147, |
| "grad_norm": 75.6977310180664, |
| "learning_rate": 1.6916080295627848e-06, |
| "logp/chosen": -868.0, |
| "logp/rejected": -1096.0, |
| "loss": 1.187255859375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.42578125, |
| "reward/margin": 3.359375, |
| "reward/rejected": -2.921875, |
| "step": 776 |
| }, |
| { |
| "approx. KL/chosen": 35.0, |
| "approx. KL/rejected": 184.0, |
| "epoch": 0.7961065573770492, |
| "grad_norm": 47.596710205078125, |
| "learning_rate": 1.6831394117609773e-06, |
| "logp/chosen": -1144.0, |
| "logp/rejected": -1712.0, |
| "loss": 0.8271484375, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.1005859375, |
| "reward/margin": 3.109375, |
| "reward/rejected": -3.203125, |
| "step": 777 |
| }, |
| { |
| "approx. KL/chosen": 71.0, |
| "approx. KL/rejected": 103.5, |
| "epoch": 0.7971311475409836, |
| "grad_norm": 93.92910766601562, |
| "learning_rate": 1.6747055778055224e-06, |
| "logp/chosen": -980.0, |
| "logp/rejected": -884.0, |
| "loss": 1.41162109375, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.2265625, |
| "reward/margin": 1.046875, |
| "reward/rejected": -2.265625, |
| "step": 778 |
| }, |
| { |
| "approx. KL/chosen": 62.5, |
| "approx. KL/rejected": 228.0, |
| "epoch": 0.798155737704918, |
| "grad_norm": 90.31454467773438, |
| "learning_rate": 1.6663066356744206e-06, |
| "logp/chosen": -764.0, |
| "logp/rejected": -948.0, |
| "loss": 1.39892578125, |
| "nll_loss": 0.4453125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.7265625, |
| "reward/margin": 1.859375, |
| "reward/rejected": -2.578125, |
| "step": 779 |
| }, |
| { |
| "approx. KL/chosen": 49.0, |
| "approx. KL/rejected": 306.0, |
| "epoch": 0.7991803278688525, |
| "grad_norm": 76.04082489013672, |
| "learning_rate": 1.6579426928989613e-06, |
| "logp/chosen": -968.0, |
| "logp/rejected": -1424.0, |
| "loss": 1.3125, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.30078125, |
| "reward/margin": 3.90625, |
| "reward/rejected": -3.609375, |
| "step": 780 |
| }, |
| { |
| "approx. KL/chosen": 11.5625, |
| "approx. KL/rejected": 300.0, |
| "epoch": 0.8002049180327869, |
| "grad_norm": 120.6811752319336, |
| "learning_rate": 1.649613856562333e-06, |
| "logp/chosen": -884.0, |
| "logp/rejected": -1008.0, |
| "loss": 1.58984375, |
| "nll_loss": 0.71875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.375, |
| "reward/margin": 1.8046875, |
| "reward/rejected": -2.171875, |
| "step": 781 |
| }, |
| { |
| "epoch": 0.8002049180327869, |
| "eval_approx. KL/chosen": 51.4125, |
| "eval_approx. KL/rejected": 232.76, |
| "eval_logp/chosen": -967.28, |
| "eval_logp/rejected": -1290.72, |
| "eval_loss": 1.3232226371765137, |
| "eval_nll_loss": 0.554453125, |
| "eval_reward/accuracy": 0.64, |
| "eval_reward/chosen": -0.552734375, |
| "eval_reward/margin": 2.2767578125, |
| "eval_reward/rejected": -2.828203125, |
| "eval_runtime": 46.6497, |
| "eval_samples_per_second": 4.244, |
| "eval_steps_per_second": 0.536, |
| "step": 781 |
| }, |
| { |
| "approx. KL/chosen": 92.0, |
| "approx. KL/rejected": 105.0, |
| "epoch": 0.8012295081967213, |
| "grad_norm": 159.55833435058594, |
| "learning_rate": 1.6413202332982607e-06, |
| "logp/chosen": -1368.0, |
| "logp/rejected": -1400.0, |
| "loss": 2.1015625, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.8046875, |
| "reward/margin": -0.0986328125, |
| "reward/rejected": -1.703125, |
| "step": 782 |
| }, |
| { |
| "approx. KL/chosen": 62.25, |
| "approx. KL/rejected": 85.0, |
| "epoch": 0.8022540983606558, |
| "grad_norm": 118.01631164550781, |
| "learning_rate": 1.6330619292896384e-06, |
| "logp/chosen": -1020.0, |
| "logp/rejected": -1320.0, |
| "loss": 1.5126953125, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.953125, |
| "reward/margin": 0.349609375, |
| "reward/rejected": -2.296875, |
| "step": 783 |
| }, |
| { |
| "approx. KL/chosen": 61.75, |
| "approx. KL/rejected": 109.5, |
| "epoch": 0.8032786885245902, |
| "grad_norm": 121.99341583251953, |
| "learning_rate": 1.6248390502671657e-06, |
| "logp/chosen": -676.0, |
| "logp/rejected": -1120.0, |
| "loss": 1.77001953125, |
| "nll_loss": 0.5078125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.0234375, |
| "reward/margin": 0.36328125, |
| "reward/rejected": -1.390625, |
| "step": 784 |
| }, |
| { |
| "approx. KL/chosen": 84.0, |
| "approx. KL/rejected": 384.0, |
| "epoch": 0.8043032786885246, |
| "grad_norm": 108.1375961303711, |
| "learning_rate": 1.6166517015080041e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1152.0, |
| "loss": 1.17919921875, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 3.625, |
| "reward/rejected": -4.03125, |
| "step": 785 |
| }, |
| { |
| "approx. KL/chosen": 11.75, |
| "approx. KL/rejected": 245.0, |
| "epoch": 0.805327868852459, |
| "grad_norm": 76.47344207763672, |
| "learning_rate": 1.6084999878344157e-06, |
| "logp/chosen": -728.0, |
| "logp/rejected": -1040.0, |
| "loss": 1.348876953125, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 2.296875, |
| "reward/rejected": -2.859375, |
| "step": 786 |
| }, |
| { |
| "approx. KL/chosen": 81.0, |
| "approx. KL/rejected": 68.0, |
| "epoch": 0.8063524590163934, |
| "grad_norm": 96.6203842163086, |
| "learning_rate": 1.6003840136124333e-06, |
| "logp/chosen": -1424.0, |
| "logp/rejected": -1296.0, |
| "loss": 1.6630859375, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.150390625, |
| "reward/margin": 1.5546875, |
| "reward/rejected": -1.40625, |
| "step": 787 |
| }, |
| { |
| "approx. KL/chosen": 36.75, |
| "approx. KL/rejected": 34.25, |
| "epoch": 0.8073770491803278, |
| "grad_norm": 89.3559799194336, |
| "learning_rate": 1.5923038827505147e-06, |
| "logp/chosen": -960.0, |
| "logp/rejected": -724.0, |
| "loss": 1.608642578125, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.82421875, |
| "reward/margin": 0.326171875, |
| "reward/rejected": -1.1484375, |
| "step": 788 |
| }, |
| { |
| "approx. KL/chosen": 47.25, |
| "approx. KL/rejected": 29.25, |
| "epoch": 0.8084016393442623, |
| "grad_norm": 107.918701171875, |
| "learning_rate": 1.5842596986982221e-06, |
| "logp/chosen": -836.0, |
| "logp/rejected": -632.0, |
| "loss": 1.71923828125, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.75, |
| "reward/margin": -0.0245361328125, |
| "reward/rejected": -0.7265625, |
| "step": 789 |
| }, |
| { |
| "approx. KL/chosen": 106.0, |
| "approx. KL/rejected": 78.0, |
| "epoch": 0.8094262295081968, |
| "grad_norm": 123.43207550048828, |
| "learning_rate": 1.576251564444887e-06, |
| "logp/chosen": -1080.0, |
| "logp/rejected": -1216.0, |
| "loss": 2.06689453125, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.828125, |
| "reward/margin": -0.400390625, |
| "reward/rejected": -1.4296875, |
| "step": 790 |
| }, |
| { |
| "approx. KL/chosen": 35.25, |
| "approx. KL/rejected": 350.0, |
| "epoch": 0.8104508196721312, |
| "grad_norm": 64.90571594238281, |
| "learning_rate": 1.5682795825183015e-06, |
| "logp/chosen": -952.0, |
| "logp/rejected": -1232.0, |
| "loss": 0.9892578125, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.953125, |
| "reward/margin": 3.21875, |
| "reward/rejected": -4.1875, |
| "step": 791 |
| }, |
| { |
| "approx. KL/chosen": 51.25, |
| "approx. KL/rejected": 320.0, |
| "epoch": 0.8114754098360656, |
| "grad_norm": 158.0845489501953, |
| "learning_rate": 1.5603438549833953e-06, |
| "logp/chosen": -1416.0, |
| "logp/rejected": -1344.0, |
| "loss": 2.02978515625, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 2.046875, |
| "reward/rejected": -2.90625, |
| "step": 792 |
| }, |
| { |
| "approx. KL/chosen": 58.5, |
| "approx. KL/rejected": 191.0, |
| "epoch": 0.8125, |
| "grad_norm": 76.18177032470703, |
| "learning_rate": 1.5524444834409423e-06, |
| "logp/chosen": -756.0, |
| "logp/rejected": -908.0, |
| "loss": 1.4619140625, |
| "nll_loss": 0.484375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.40234375, |
| "reward/margin": 2.140625, |
| "reward/rejected": -2.546875, |
| "step": 793 |
| }, |
| { |
| "approx. KL/chosen": 16.0, |
| "approx. KL/rejected": 71.5, |
| "epoch": 0.8135245901639344, |
| "grad_norm": 53.80184555053711, |
| "learning_rate": 1.5445815690262457e-06, |
| "logp/chosen": -756.0, |
| "logp/rejected": -1240.0, |
| "loss": 1.15625, |
| "nll_loss": 0.484375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.474609375, |
| "reward/margin": 1.3046875, |
| "reward/rejected": -0.828125, |
| "step": 794 |
| }, |
| { |
| "approx. KL/chosen": 31.75, |
| "approx. KL/rejected": 193.0, |
| "epoch": 0.8145491803278688, |
| "grad_norm": 80.46125793457031, |
| "learning_rate": 1.536755212407853e-06, |
| "logp/chosen": -980.0, |
| "logp/rejected": -1144.0, |
| "loss": 1.608154296875, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.6484375, |
| "reward/margin": 2.0625, |
| "reward/rejected": -2.703125, |
| "step": 795 |
| }, |
| { |
| "approx. KL/chosen": 33.0, |
| "approx. KL/rejected": 172.0, |
| "epoch": 0.8155737704918032, |
| "grad_norm": 112.89042663574219, |
| "learning_rate": 1.5289655137862618e-06, |
| "logp/chosen": -1328.0, |
| "logp/rejected": -1496.0, |
| "loss": 1.42041015625, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 1.703125, |
| "reward/rejected": -1.796875, |
| "step": 796 |
| }, |
| { |
| "approx. KL/chosen": 37.25, |
| "approx. KL/rejected": 50.5, |
| "epoch": 0.8165983606557377, |
| "grad_norm": 37.881813049316406, |
| "learning_rate": 1.521212572892641e-06, |
| "logp/chosen": -736.0, |
| "logp/rejected": -928.0, |
| "loss": 1.02734375, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.376953125, |
| "reward/margin": 1.8515625, |
| "reward/rejected": -1.4765625, |
| "step": 797 |
| }, |
| { |
| "approx. KL/chosen": 29.25, |
| "approx. KL/rejected": 456.0, |
| "epoch": 0.8176229508196722, |
| "grad_norm": 51.26923751831055, |
| "learning_rate": 1.5134964889875516e-06, |
| "logp/chosen": -864.0, |
| "logp/rejected": -1464.0, |
| "loss": 1.1015625, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 4.25, |
| "reward/rejected": -4.40625, |
| "step": 798 |
| }, |
| { |
| "approx. KL/chosen": 89.0, |
| "approx. KL/rejected": 278.0, |
| "epoch": 0.8186475409836066, |
| "grad_norm": 109.20646667480469, |
| "learning_rate": 1.5058173608596763e-06, |
| "logp/chosen": -1020.0, |
| "logp/rejected": -1256.0, |
| "loss": 1.510009765625, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.25, |
| "reward/margin": 2.328125, |
| "reward/rejected": -3.578125, |
| "step": 799 |
| }, |
| { |
| "approx. KL/chosen": 75.5, |
| "approx. KL/rejected": 229.0, |
| "epoch": 0.819672131147541, |
| "grad_norm": 75.65728759765625, |
| "learning_rate": 1.4981752868245517e-06, |
| "logp/chosen": -556.0, |
| "logp/rejected": -984.0, |
| "loss": 1.66650390625, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.453125, |
| "reward/margin": 1.4765625, |
| "reward/rejected": -2.9375, |
| "step": 800 |
| }, |
| { |
| "approx. KL/chosen": 35.5, |
| "approx. KL/rejected": 241.0, |
| "epoch": 0.8206967213114754, |
| "grad_norm": 85.89702606201172, |
| "learning_rate": 1.4905703647233158e-06, |
| "logp/chosen": -1136.0, |
| "logp/rejected": -1408.0, |
| "loss": 1.14111328125, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": 0.625, |
| "reward/margin": 2.71875, |
| "reward/rejected": -2.109375, |
| "step": 801 |
| }, |
| { |
| "approx. KL/chosen": 57.0, |
| "approx. KL/rejected": 146.0, |
| "epoch": 0.8217213114754098, |
| "grad_norm": 68.47370147705078, |
| "learning_rate": 1.4830026919214517e-06, |
| "logp/chosen": -964.0, |
| "logp/rejected": -1424.0, |
| "loss": 0.860107421875, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.27734375, |
| "reward/margin": 3.078125, |
| "reward/rejected": -2.796875, |
| "step": 802 |
| }, |
| { |
| "approx. KL/chosen": 58.5, |
| "approx. KL/rejected": 334.0, |
| "epoch": 0.8227459016393442, |
| "grad_norm": 31.64858055114746, |
| "learning_rate": 1.4754723653075415e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1672.0, |
| "loss": 0.72265625, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.67578125, |
| "reward/margin": 4.71875, |
| "reward/rejected": -4.03125, |
| "step": 803 |
| }, |
| { |
| "approx. KL/chosen": 20.25, |
| "approx. KL/rejected": 165.0, |
| "epoch": 0.8237704918032787, |
| "grad_norm": 114.01292419433594, |
| "learning_rate": 1.4679794812920228e-06, |
| "logp/chosen": -836.0, |
| "logp/rejected": -1384.0, |
| "loss": 1.4609375, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 1.6484375, |
| "reward/rejected": -2.296875, |
| "step": 804 |
| }, |
| { |
| "approx. KL/chosen": 14.5, |
| "approx. KL/rejected": 216.0, |
| "epoch": 0.8247950819672131, |
| "grad_norm": 63.002376556396484, |
| "learning_rate": 1.4605241358059597e-06, |
| "logp/chosen": -1192.0, |
| "logp/rejected": -1368.0, |
| "loss": 0.83349609375, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 3.046875, |
| "reward/rejected": -3.359375, |
| "step": 805 |
| }, |
| { |
| "approx. KL/chosen": 39.0, |
| "approx. KL/rejected": 127.0, |
| "epoch": 0.8258196721311475, |
| "grad_norm": 98.99922180175781, |
| "learning_rate": 1.453106424299813e-06, |
| "logp/chosen": -940.0, |
| "logp/rejected": -1152.0, |
| "loss": 1.10498046875, |
| "nll_loss": 0.486328125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 1.6015625, |
| "reward/rejected": -2.5, |
| "step": 806 |
| }, |
| { |
| "approx. KL/chosen": 26.0, |
| "approx. KL/rejected": 152.0, |
| "epoch": 0.826844262295082, |
| "grad_norm": 102.27841186523438, |
| "learning_rate": 1.445726441742216e-06, |
| "logp/chosen": -1064.0, |
| "logp/rejected": -1192.0, |
| "loss": 1.63916015625, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 1.8515625, |
| "reward/rejected": -2.046875, |
| "step": 807 |
| }, |
| { |
| "approx. KL/chosen": 61.0, |
| "approx. KL/rejected": 169.0, |
| "epoch": 0.8278688524590164, |
| "grad_norm": 123.59063720703125, |
| "learning_rate": 1.4383842826187586e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1760.0, |
| "loss": 1.796875, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.326171875, |
| "reward/margin": 1.5, |
| "reward/rejected": -1.171875, |
| "step": 808 |
| }, |
| { |
| "approx. KL/chosen": 44.75, |
| "approx. KL/rejected": 62.0, |
| "epoch": 0.8288934426229508, |
| "grad_norm": 82.93122863769531, |
| "learning_rate": 1.431080040930782e-06, |
| "logp/chosen": -466.0, |
| "logp/rejected": -628.0, |
| "loss": 1.496826171875, |
| "nll_loss": 0.46484375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.2734375, |
| "reward/margin": 0.42578125, |
| "reward/rejected": -1.703125, |
| "step": 809 |
| }, |
| { |
| "approx. KL/chosen": 36.5, |
| "approx. KL/rejected": 150.0, |
| "epoch": 0.8299180327868853, |
| "grad_norm": 55.160911560058594, |
| "learning_rate": 1.42381381019417e-06, |
| "logp/chosen": -908.0, |
| "logp/rejected": -1080.0, |
| "loss": 1.037109375, |
| "nll_loss": 0.6640625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.078125, |
| "reward/margin": 1.7890625, |
| "reward/rejected": -2.859375, |
| "step": 810 |
| }, |
| { |
| "approx. KL/chosen": 24.25, |
| "approx. KL/rejected": 564.0, |
| "epoch": 0.8309426229508197, |
| "grad_norm": 67.18357849121094, |
| "learning_rate": 1.4165856834381547e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -1704.0, |
| "loss": 1.08544921875, |
| "nll_loss": 0.640625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.953125, |
| "reward/margin": 4.21875, |
| "reward/rejected": -5.15625, |
| "step": 811 |
| }, |
| { |
| "approx. KL/chosen": 25.75, |
| "approx. KL/rejected": 231.0, |
| "epoch": 0.8319672131147541, |
| "grad_norm": 73.38434600830078, |
| "learning_rate": 1.4093957532041235e-06, |
| "logp/chosen": -820.0, |
| "logp/rejected": -792.0, |
| "loss": 1.1396484375, |
| "nll_loss": 0.47265625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.0751953125, |
| "reward/margin": 2.453125, |
| "reward/rejected": -2.375, |
| "step": 812 |
| }, |
| { |
| "approx. KL/chosen": 27.5, |
| "approx. KL/rejected": 134.0, |
| "epoch": 0.8329918032786885, |
| "grad_norm": 86.10533142089844, |
| "learning_rate": 1.4022441115444367e-06, |
| "logp/chosen": -980.0, |
| "logp/rejected": -1152.0, |
| "loss": 1.422607421875, |
| "nll_loss": 0.5078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.69921875, |
| "reward/margin": 1.203125, |
| "reward/rejected": -1.8984375, |
| "step": 813 |
| }, |
| { |
| "approx. KL/chosen": 35.25, |
| "approx. KL/rejected": 138.0, |
| "epoch": 0.8340163934426229, |
| "grad_norm": 44.21250915527344, |
| "learning_rate": 1.3951308500212473e-06, |
| "logp/chosen": -808.0, |
| "logp/rejected": -1280.0, |
| "loss": 1.07177734375, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.57421875, |
| "reward/margin": 3.03125, |
| "reward/rejected": -2.453125, |
| "step": 814 |
| }, |
| { |
| "approx. KL/chosen": 44.0, |
| "approx. KL/rejected": 199.0, |
| "epoch": 0.8350409836065574, |
| "grad_norm": 102.99156951904297, |
| "learning_rate": 1.3880560597053301e-06, |
| "logp/chosen": -868.0, |
| "logp/rejected": -964.0, |
| "loss": 0.88916015625, |
| "nll_loss": 0.46875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 2.90625, |
| "reward/rejected": -3.5, |
| "step": 815 |
| }, |
| { |
| "approx. KL/chosen": 67.0, |
| "approx. KL/rejected": 164.0, |
| "epoch": 0.8360655737704918, |
| "grad_norm": 93.47496795654297, |
| "learning_rate": 1.3810198311749115e-06, |
| "logp/chosen": -1120.0, |
| "logp/rejected": -1512.0, |
| "loss": 1.16796875, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.5, |
| "reward/margin": 2.5, |
| "reward/rejected": -3.0, |
| "step": 816 |
| }, |
| { |
| "approx. KL/chosen": 26.0, |
| "approx. KL/rejected": 172.0, |
| "epoch": 0.8370901639344263, |
| "grad_norm": 51.12922668457031, |
| "learning_rate": 1.3740222545145165e-06, |
| "logp/chosen": -912.0, |
| "logp/rejected": -1208.0, |
| "loss": 1.09619140625, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 2.3125, |
| "reward/rejected": -2.921875, |
| "step": 817 |
| }, |
| { |
| "approx. KL/chosen": 121.0, |
| "approx. KL/rejected": 135.0, |
| "epoch": 0.8381147540983607, |
| "grad_norm": 112.27657318115234, |
| "learning_rate": 1.3670634193138104e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1136.0, |
| "loss": 1.99755859375, |
| "nll_loss": 0.71875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.75, |
| "reward/margin": 0.75390625, |
| "reward/rejected": -2.5, |
| "step": 818 |
| }, |
| { |
| "approx. KL/chosen": 10.75, |
| "approx. KL/rejected": 211.0, |
| "epoch": 0.8391393442622951, |
| "grad_norm": 35.41783142089844, |
| "learning_rate": 1.3601434146664545e-06, |
| "logp/chosen": -724.0, |
| "logp/rejected": -904.0, |
| "loss": 0.90966796875, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.25, |
| "reward/margin": 3.015625, |
| "reward/rejected": -2.765625, |
| "step": 819 |
| }, |
| { |
| "approx. KL/chosen": 61.25, |
| "approx. KL/rejected": 282.0, |
| "epoch": 0.8401639344262295, |
| "grad_norm": 128.46322631835938, |
| "learning_rate": 1.3532623291689615e-06, |
| "logp/chosen": -964.0, |
| "logp/rejected": -1680.0, |
| "loss": 1.69921875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.1513671875, |
| "reward/margin": 2.84375, |
| "reward/rejected": -3.0, |
| "step": 820 |
| }, |
| { |
| "approx. KL/chosen": 52.0, |
| "approx. KL/rejected": 174.0, |
| "epoch": 0.8411885245901639, |
| "grad_norm": 70.52205657958984, |
| "learning_rate": 1.3464202509195675e-06, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1736.0, |
| "loss": 1.17578125, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.201171875, |
| "reward/margin": 3.0, |
| "reward/rejected": -3.203125, |
| "step": 821 |
| }, |
| { |
| "approx. KL/chosen": 51.0, |
| "approx. KL/rejected": 99.5, |
| "epoch": 0.8422131147540983, |
| "grad_norm": 81.26568603515625, |
| "learning_rate": 1.3396172675170955e-06, |
| "logp/chosen": -624.0, |
| "logp/rejected": -632.0, |
| "loss": 1.119140625, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.9765625, |
| "reward/margin": 1.546875, |
| "reward/rejected": -2.53125, |
| "step": 822 |
| }, |
| { |
| "approx. KL/chosen": 121.5, |
| "approx. KL/rejected": 138.0, |
| "epoch": 0.8432377049180327, |
| "grad_norm": 109.61383819580078, |
| "learning_rate": 1.3328534660598462e-06, |
| "logp/chosen": -1256.0, |
| "logp/rejected": -1312.0, |
| "loss": 2.01171875, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.4296875, |
| "reward/margin": 0.77734375, |
| "reward/rejected": -2.203125, |
| "step": 823 |
| }, |
| { |
| "approx. KL/chosen": 129.0, |
| "approx. KL/rejected": 326.0, |
| "epoch": 0.8442622950819673, |
| "grad_norm": 125.98746490478516, |
| "learning_rate": 1.3261289331444694e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1032.0, |
| "loss": 1.984375, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.25, |
| "reward/margin": 2.15625, |
| "reward/rejected": -3.40625, |
| "step": 824 |
| }, |
| { |
| "approx. KL/chosen": 42.0, |
| "approx. KL/rejected": 362.0, |
| "epoch": 0.8452868852459017, |
| "grad_norm": 83.95372009277344, |
| "learning_rate": 1.3194437548648644e-06, |
| "logp/chosen": -1208.0, |
| "logp/rejected": -1560.0, |
| "loss": 1.3427734375, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 3.453125, |
| "reward/rejected": -4.0625, |
| "step": 825 |
| }, |
| { |
| "approx. KL/chosen": 42.0, |
| "approx. KL/rejected": 153.0, |
| "epoch": 0.8463114754098361, |
| "grad_norm": 72.95390319824219, |
| "learning_rate": 1.3127980168110715e-06, |
| "logp/chosen": -1024.0, |
| "logp/rejected": -1392.0, |
| "loss": 1.0703125, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": 0.400390625, |
| "reward/margin": 2.046875, |
| "reward/rejected": -1.6484375, |
| "step": 826 |
| }, |
| { |
| "approx. KL/chosen": 22.0, |
| "approx. KL/rejected": 44.0, |
| "epoch": 0.8473360655737705, |
| "grad_norm": 57.19609069824219, |
| "learning_rate": 1.3061918040681862e-06, |
| "logp/chosen": -828.0, |
| "logp/rejected": -952.0, |
| "loss": 1.111328125, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.21875, |
| "reward/margin": 1.3125, |
| "reward/rejected": -1.09375, |
| "step": 827 |
| }, |
| { |
| "approx. KL/chosen": 14.0, |
| "approx. KL/rejected": 156.0, |
| "epoch": 0.8483606557377049, |
| "grad_norm": 82.22496795654297, |
| "learning_rate": 1.2996252012152557e-06, |
| "logp/chosen": -968.0, |
| "logp/rejected": -1328.0, |
| "loss": 1.10693359375, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 2.0, |
| "reward/rejected": -2.609375, |
| "step": 828 |
| }, |
| { |
| "approx. KL/chosen": 17.25, |
| "approx. KL/rejected": 235.0, |
| "epoch": 0.8493852459016393, |
| "grad_norm": 53.802032470703125, |
| "learning_rate": 1.2930982923242092e-06, |
| "logp/chosen": -936.0, |
| "logp/rejected": -1040.0, |
| "loss": 1.01611328125, |
| "nll_loss": 0.6796875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 3.125, |
| "reward/rejected": -3.265625, |
| "step": 829 |
| }, |
| { |
| "approx. KL/chosen": 16.25, |
| "approx. KL/rejected": 227.0, |
| "epoch": 0.8504098360655737, |
| "grad_norm": 111.5277328491211, |
| "learning_rate": 1.286611160958771e-06, |
| "logp/chosen": -1200.0, |
| "logp/rejected": -1184.0, |
| "loss": 1.26513671875, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 1.796875, |
| "reward/rejected": -1.9453125, |
| "step": 830 |
| }, |
| { |
| "approx. KL/chosen": 25.25, |
| "approx. KL/rejected": 304.0, |
| "epoch": 0.8514344262295082, |
| "grad_norm": 33.6314697265625, |
| "learning_rate": 1.2801638901733996e-06, |
| "logp/chosen": -632.0, |
| "logp/rejected": -1184.0, |
| "loss": 0.86767578125, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.1494140625, |
| "reward/margin": 3.859375, |
| "reward/rejected": -4.0, |
| "step": 831 |
| }, |
| { |
| "approx. KL/chosen": 11.25, |
| "approx. KL/rejected": 193.0, |
| "epoch": 0.8524590163934426, |
| "grad_norm": 52.833030700683594, |
| "learning_rate": 1.2737565625122192e-06, |
| "logp/chosen": -828.0, |
| "logp/rejected": -1128.0, |
| "loss": 1.038818359375, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.34375, |
| "reward/margin": 1.9765625, |
| "reward/rejected": -2.3125, |
| "step": 832 |
| }, |
| { |
| "approx. KL/chosen": 17.75, |
| "approx. KL/rejected": 61.5, |
| "epoch": 0.8534836065573771, |
| "grad_norm": 138.54678344726562, |
| "learning_rate": 1.2673892600079623e-06, |
| "logp/chosen": -880.0, |
| "logp/rejected": -1056.0, |
| "loss": 1.2646484375, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.025146484375, |
| "reward/margin": 0.86328125, |
| "reward/rejected": -0.88671875, |
| "step": 833 |
| }, |
| { |
| "approx. KL/chosen": 7.0, |
| "approx. KL/rejected": 314.0, |
| "epoch": 0.8545081967213115, |
| "grad_norm": 33.57254409790039, |
| "learning_rate": 1.2610620641809236e-06, |
| "logp/chosen": -756.0, |
| "logp/rejected": -1020.0, |
| "loss": 0.74755859375, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 4.34375, |
| "reward/rejected": -4.4375, |
| "step": 834 |
| }, |
| { |
| "approx. KL/chosen": 15.5625, |
| "approx. KL/rejected": 47.0, |
| "epoch": 0.8555327868852459, |
| "grad_norm": 96.47374725341797, |
| "learning_rate": 1.2547750560379094e-06, |
| "logp/chosen": -948.0, |
| "logp/rejected": -868.0, |
| "loss": 1.245849609375, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.5234375, |
| "reward/margin": 0.77734375, |
| "reward/rejected": -1.3046875, |
| "step": 835 |
| }, |
| { |
| "approx. KL/chosen": 91.0, |
| "approx. KL/rejected": 150.0, |
| "epoch": 0.8565573770491803, |
| "grad_norm": 109.43460083007812, |
| "learning_rate": 1.2485283160712118e-06, |
| "logp/chosen": -784.0, |
| "logp/rejected": -524.0, |
| "loss": 1.87548828125, |
| "nll_loss": 0.515625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.51171875, |
| "reward/margin": 1.46875, |
| "reward/rejected": -1.984375, |
| "step": 836 |
| }, |
| { |
| "approx. KL/chosen": 24.75, |
| "approx. KL/rejected": 302.0, |
| "epoch": 0.8575819672131147, |
| "grad_norm": 75.6491470336914, |
| "learning_rate": 1.2423219242575637e-06, |
| "logp/chosen": -664.0, |
| "logp/rejected": -1208.0, |
| "loss": 0.982666015625, |
| "nll_loss": 0.474609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.474609375, |
| "reward/margin": 3.171875, |
| "reward/rejected": -3.640625, |
| "step": 837 |
| }, |
| { |
| "approx. KL/chosen": 24.25, |
| "approx. KL/rejected": 128.0, |
| "epoch": 0.8586065573770492, |
| "grad_norm": 117.73612213134766, |
| "learning_rate": 1.2361559600571284e-06, |
| "logp/chosen": -1144.0, |
| "logp/rejected": -1360.0, |
| "loss": 1.39013671875, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 1.1015625, |
| "reward/rejected": -1.6484375, |
| "step": 838 |
| }, |
| { |
| "approx. KL/chosen": 48.25, |
| "approx. KL/rejected": 44.0, |
| "epoch": 0.8596311475409836, |
| "grad_norm": 52.65054702758789, |
| "learning_rate": 1.2300305024124716e-06, |
| "logp/chosen": -976.0, |
| "logp/rejected": -1192.0, |
| "loss": 0.952392578125, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.349609375, |
| "reward/margin": 1.7265625, |
| "reward/rejected": -1.375, |
| "step": 839 |
| }, |
| { |
| "approx. KL/chosen": 52.25, |
| "approx. KL/rejected": 101.5, |
| "epoch": 0.860655737704918, |
| "grad_norm": 119.9682388305664, |
| "learning_rate": 1.2239456297475596e-06, |
| "logp/chosen": -1224.0, |
| "logp/rejected": -1408.0, |
| "loss": 1.50048828125, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 1.1875, |
| "reward/rejected": -1.8359375, |
| "step": 840 |
| }, |
| { |
| "approx. KL/chosen": 21.5, |
| "approx. KL/rejected": 398.0, |
| "epoch": 0.8616803278688525, |
| "grad_norm": 76.29112243652344, |
| "learning_rate": 1.2179014199667477e-06, |
| "logp/chosen": -852.0, |
| "logp/rejected": -1424.0, |
| "loss": 1.0634765625, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 4.03125, |
| "reward/rejected": -4.21875, |
| "step": 841 |
| }, |
| { |
| "approx. KL/chosen": 45.0, |
| "approx. KL/rejected": 772.0, |
| "epoch": 0.8627049180327869, |
| "grad_norm": 29.902883529663086, |
| "learning_rate": 1.2118979504537883e-06, |
| "logp/chosen": -1120.0, |
| "logp/rejected": -1712.0, |
| "loss": 0.80078125, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -1.3046875, |
| "reward/margin": 4.4375, |
| "reward/rejected": -5.75, |
| "step": 842 |
| }, |
| { |
| "approx. KL/chosen": 91.5, |
| "approx. KL/rejected": 78.5, |
| "epoch": 0.8637295081967213, |
| "grad_norm": 121.99256896972656, |
| "learning_rate": 1.2059352980708351e-06, |
| "logp/chosen": -860.0, |
| "logp/rejected": -880.0, |
| "loss": 1.8349609375, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.90625, |
| "reward/margin": 0.35546875, |
| "reward/rejected": -2.25, |
| "step": 843 |
| }, |
| { |
| "approx. KL/chosen": 39.25, |
| "approx. KL/rejected": 150.0, |
| "epoch": 0.8647540983606558, |
| "grad_norm": 105.86071014404297, |
| "learning_rate": 1.2000135391574654e-06, |
| "logp/chosen": -880.0, |
| "logp/rejected": -1128.0, |
| "loss": 1.2021484375, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 1.8515625, |
| "reward/rejected": -2.40625, |
| "step": 844 |
| }, |
| { |
| "approx. KL/chosen": 22.0, |
| "approx. KL/rejected": 27.0, |
| "epoch": 0.8657786885245902, |
| "grad_norm": 119.51807403564453, |
| "learning_rate": 1.1941327495296965e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1288.0, |
| "loss": 1.720703125, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 0.1005859375, |
| "reward/rejected": -0.5, |
| "step": 845 |
| }, |
| { |
| "approx. KL/chosen": 28.0, |
| "approx. KL/rejected": 203.0, |
| "epoch": 0.8668032786885246, |
| "grad_norm": 72.1602783203125, |
| "learning_rate": 1.1882930044790202e-06, |
| "logp/chosen": -928.0, |
| "logp/rejected": -1144.0, |
| "loss": 0.839111328125, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 1.0, |
| "reward/margin": 3.75, |
| "reward/rejected": -2.75, |
| "step": 846 |
| }, |
| { |
| "approx. KL/chosen": 25.0, |
| "approx. KL/rejected": 268.0, |
| "epoch": 0.867827868852459, |
| "grad_norm": 59.034217834472656, |
| "learning_rate": 1.1824943787714332e-06, |
| "logp/chosen": -1128.0, |
| "logp/rejected": -1856.0, |
| "loss": 0.9765625, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 3.703125, |
| "reward/rejected": -3.8125, |
| "step": 847 |
| }, |
| { |
| "approx. KL/chosen": 125.0, |
| "approx. KL/rejected": 242.0, |
| "epoch": 0.8688524590163934, |
| "grad_norm": 105.83798217773438, |
| "learning_rate": 1.1767369466464857e-06, |
| "logp/chosen": -1264.0, |
| "logp/rejected": -1368.0, |
| "loss": 2.29296875, |
| "nll_loss": 0.66796875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 1.90625, |
| "reward/rejected": -3.0, |
| "step": 848 |
| }, |
| { |
| "approx. KL/chosen": 56.25, |
| "approx. KL/rejected": 133.0, |
| "epoch": 0.8698770491803278, |
| "grad_norm": 134.82028198242188, |
| "learning_rate": 1.1710207818163276e-06, |
| "logp/chosen": -1056.0, |
| "logp/rejected": -1608.0, |
| "loss": 1.8935546875, |
| "nll_loss": 0.671875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.451171875, |
| "reward/margin": 0.44921875, |
| "reward/rejected": -0.90234375, |
| "step": 849 |
| }, |
| { |
| "approx. KL/chosen": 83.5, |
| "approx. KL/rejected": 175.0, |
| "epoch": 0.8709016393442623, |
| "grad_norm": 147.7592315673828, |
| "learning_rate": 1.1653459574647653e-06, |
| "logp/chosen": -1464.0, |
| "logp/rejected": -1376.0, |
| "loss": 2.004638671875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.625, |
| "reward/margin": 1.6640625, |
| "reward/rejected": -2.296875, |
| "step": 850 |
| }, |
| { |
| "approx. KL/chosen": 31.0, |
| "approx. KL/rejected": 102.0, |
| "epoch": 0.8719262295081968, |
| "grad_norm": 84.67029571533203, |
| "learning_rate": 1.1597125462463238e-06, |
| "logp/chosen": -1152.0, |
| "logp/rejected": -1256.0, |
| "loss": 1.06689453125, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.1015625, |
| "reward/margin": 1.15625, |
| "reward/rejected": -2.25, |
| "step": 851 |
| }, |
| { |
| "approx. KL/chosen": 11.5, |
| "approx. KL/rejected": 97.0, |
| "epoch": 0.8729508196721312, |
| "grad_norm": 70.06684112548828, |
| "learning_rate": 1.1541206202853194e-06, |
| "logp/chosen": -732.0, |
| "logp/rejected": -912.0, |
| "loss": 1.20947265625, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 1.546875, |
| "reward/rejected": -1.8515625, |
| "step": 852 |
| }, |
| { |
| "approx. KL/chosen": 14.75, |
| "approx. KL/rejected": 97.0, |
| "epoch": 0.8739754098360656, |
| "grad_norm": 69.1263427734375, |
| "learning_rate": 1.1485702511749338e-06, |
| "logp/chosen": -964.0, |
| "logp/rejected": -1192.0, |
| "loss": 0.88623046875, |
| "nll_loss": 0.451171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.322265625, |
| "reward/margin": 2.03125, |
| "reward/rejected": -1.703125, |
| "step": 853 |
| }, |
| { |
| "approx. KL/chosen": 46.0, |
| "approx. KL/rejected": 217.0, |
| "epoch": 0.875, |
| "grad_norm": 112.37863159179688, |
| "learning_rate": 1.1430615099762984e-06, |
| "logp/chosen": -1040.0, |
| "logp/rejected": -1744.0, |
| "loss": 1.18896484375, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.0, |
| "reward/margin": 2.109375, |
| "reward/rejected": -3.109375, |
| "step": 854 |
| }, |
| { |
| "approx. KL/chosen": 25.5, |
| "approx. KL/rejected": 169.0, |
| "epoch": 0.8760245901639344, |
| "grad_norm": 50.90610122680664, |
| "learning_rate": 1.1375944672175823e-06, |
| "logp/chosen": -916.0, |
| "logp/rejected": -1008.0, |
| "loss": 0.840576171875, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.42578125, |
| "reward/margin": 2.3125, |
| "reward/rejected": -2.734375, |
| "step": 855 |
| }, |
| { |
| "approx. KL/chosen": 55.0, |
| "approx. KL/rejected": 83.5, |
| "epoch": 0.8770491803278688, |
| "grad_norm": 93.8719711303711, |
| "learning_rate": 1.132169192893095e-06, |
| "logp/chosen": -708.0, |
| "logp/rejected": -816.0, |
| "loss": 1.47607421875, |
| "nll_loss": 0.64453125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.1796875, |
| "reward/margin": 0.91015625, |
| "reward/rejected": -2.09375, |
| "step": 856 |
| }, |
| { |
| "approx. KL/chosen": 7.5625, |
| "approx. KL/rejected": 70.0, |
| "epoch": 0.8780737704918032, |
| "grad_norm": 48.08904266357422, |
| "learning_rate": 1.1267857564623827e-06, |
| "logp/chosen": -510.0, |
| "logp/rejected": -624.0, |
| "loss": 1.1982421875, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.025390625, |
| "reward/margin": 1.4296875, |
| "reward/rejected": -1.453125, |
| "step": 857 |
| }, |
| { |
| "approx. KL/chosen": 13.25, |
| "approx. KL/rejected": 64.5, |
| "epoch": 0.8790983606557377, |
| "grad_norm": 66.1580581665039, |
| "learning_rate": 1.121444226849347e-06, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1216.0, |
| "loss": 0.962890625, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.150390625, |
| "reward/margin": 1.9765625, |
| "reward/rejected": -1.828125, |
| "step": 858 |
| }, |
| { |
| "approx. KL/chosen": 80.0, |
| "approx. KL/rejected": 174.0, |
| "epoch": 0.8801229508196722, |
| "grad_norm": 120.88162994384766, |
| "learning_rate": 1.1161446724413536e-06, |
| "logp/chosen": -1232.0, |
| "logp/rejected": -1640.0, |
| "loss": 1.888427734375, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.2890625, |
| "reward/margin": 0.8828125, |
| "reward/rejected": -1.171875, |
| "step": 859 |
| }, |
| { |
| "approx. KL/chosen": 35.0, |
| "approx. KL/rejected": 135.0, |
| "epoch": 0.8811475409836066, |
| "grad_norm": 75.09941101074219, |
| "learning_rate": 1.110887161088365e-06, |
| "logp/chosen": -788.0, |
| "logp/rejected": -1136.0, |
| "loss": 1.068359375, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 1.828125, |
| "reward/rejected": -2.484375, |
| "step": 860 |
| }, |
| { |
| "approx. KL/chosen": 70.0, |
| "approx. KL/rejected": 29.0, |
| "epoch": 0.882172131147541, |
| "grad_norm": 110.83566284179688, |
| "learning_rate": 1.1056717601020676e-06, |
| "logp/chosen": -1160.0, |
| "logp/rejected": -928.0, |
| "loss": 1.662841796875, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.875, |
| "reward/margin": 0.052734375, |
| "reward/rejected": -0.92578125, |
| "step": 861 |
| }, |
| { |
| "approx. KL/chosen": 65.0, |
| "approx. KL/rejected": 50.75, |
| "epoch": 0.8831967213114754, |
| "grad_norm": 72.66386413574219, |
| "learning_rate": 1.1004985362550106e-06, |
| "logp/chosen": -872.0, |
| "logp/rejected": -1112.0, |
| "loss": 1.2998046875, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.251953125, |
| "reward/margin": 1.5703125, |
| "reward/rejected": -1.8203125, |
| "step": 862 |
| }, |
| { |
| "approx. KL/chosen": 32.25, |
| "approx. KL/rejected": 502.0, |
| "epoch": 0.8842213114754098, |
| "grad_norm": 41.236881256103516, |
| "learning_rate": 1.0953675557797498e-06, |
| "logp/chosen": -716.0, |
| "logp/rejected": -924.0, |
| "loss": 0.9189453125, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.7734375, |
| "reward/margin": 3.453125, |
| "reward/rejected": -4.21875, |
| "step": 863 |
| }, |
| { |
| "approx. KL/chosen": 31.125, |
| "approx. KL/rejected": 294.0, |
| "epoch": 0.8852459016393442, |
| "grad_norm": 49.09882736206055, |
| "learning_rate": 1.0902788843680024e-06, |
| "logp/chosen": -776.0, |
| "logp/rejected": -1336.0, |
| "loss": 1.103759765625, |
| "nll_loss": 0.67578125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.349609375, |
| "reward/margin": 4.03125, |
| "reward/rejected": -3.671875, |
| "step": 864 |
| }, |
| { |
| "approx. KL/chosen": 15.25, |
| "approx. KL/rejected": 286.0, |
| "epoch": 0.8862704918032787, |
| "grad_norm": 47.453765869140625, |
| "learning_rate": 1.0852325871698044e-06, |
| "logp/chosen": -944.0, |
| "logp/rejected": -1608.0, |
| "loss": 0.85986328125, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.150390625, |
| "reward/margin": 4.15625, |
| "reward/rejected": -4.0, |
| "step": 865 |
| }, |
| { |
| "approx. KL/chosen": 27.5, |
| "approx. KL/rejected": 229.0, |
| "epoch": 0.8872950819672131, |
| "grad_norm": 54.392513275146484, |
| "learning_rate": 1.0802287287926772e-06, |
| "logp/chosen": -760.0, |
| "logp/rejected": -968.0, |
| "loss": 1.1396484375, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 3.015625, |
| "reward/rejected": -3.421875, |
| "step": 866 |
| }, |
| { |
| "approx. KL/chosen": 45.75, |
| "approx. KL/rejected": 380.0, |
| "epoch": 0.8883196721311475, |
| "grad_norm": 62.97100830078125, |
| "learning_rate": 1.0752673733007975e-06, |
| "logp/chosen": -800.0, |
| "logp/rejected": -1104.0, |
| "loss": 1.110107421875, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.0751953125, |
| "reward/margin": 4.03125, |
| "reward/rejected": -4.09375, |
| "step": 867 |
| }, |
| { |
| "approx. KL/chosen": 26.0, |
| "approx. KL/rejected": 81.0, |
| "epoch": 0.889344262295082, |
| "grad_norm": 80.13520050048828, |
| "learning_rate": 1.0703485842141816e-06, |
| "logp/chosen": -1168.0, |
| "logp/rejected": -1040.0, |
| "loss": 1.15185546875, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 0.9765625, |
| "reward/rejected": -1.171875, |
| "step": 868 |
| }, |
| { |
| "approx. KL/chosen": 58.25, |
| "approx. KL/rejected": 356.0, |
| "epoch": 0.8903688524590164, |
| "grad_norm": 77.51178741455078, |
| "learning_rate": 1.0654724245078694e-06, |
| "logp/chosen": -748.0, |
| "logp/rejected": -1072.0, |
| "loss": 1.423095703125, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 3.0, |
| "reward/rejected": -3.546875, |
| "step": 869 |
| }, |
| { |
| "approx. KL/chosen": 70.0, |
| "approx. KL/rejected": 77.0, |
| "epoch": 0.8913934426229508, |
| "grad_norm": 115.84623718261719, |
| "learning_rate": 1.0606389566111196e-06, |
| "logp/chosen": -1200.0, |
| "logp/rejected": -1392.0, |
| "loss": 1.32861328125, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.201171875, |
| "reward/margin": 1.6171875, |
| "reward/rejected": -1.8125, |
| "step": 870 |
| }, |
| { |
| "approx. KL/chosen": 5.625, |
| "approx. KL/rejected": 186.0, |
| "epoch": 0.8924180327868853, |
| "grad_norm": 64.05183410644531, |
| "learning_rate": 1.0558482424066072e-06, |
| "logp/chosen": -704.0, |
| "logp/rejected": -996.0, |
| "loss": 1.1494140625, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 1.875, |
| "reward/rejected": -2.28125, |
| "step": 871 |
| }, |
| { |
| "approx. KL/chosen": 23.25, |
| "approx. KL/rejected": 140.0, |
| "epoch": 0.8934426229508197, |
| "grad_norm": 93.31915283203125, |
| "learning_rate": 1.0511003432296356e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -1064.0, |
| "loss": 1.22216796875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 2.0, |
| "reward/rejected": -2.15625, |
| "step": 872 |
| }, |
| { |
| "approx. KL/chosen": 38.5, |
| "approx. KL/rejected": 280.0, |
| "epoch": 0.8944672131147541, |
| "grad_norm": 11.988310813903809, |
| "learning_rate": 1.0463953198673472e-06, |
| "logp/chosen": -856.0, |
| "logp/rejected": -1264.0, |
| "loss": 0.758544921875, |
| "nll_loss": 0.6953125, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.375, |
| "reward/margin": 3.8125, |
| "reward/rejected": -4.1875, |
| "step": 873 |
| }, |
| { |
| "approx. KL/chosen": 22.5, |
| "approx. KL/rejected": 228.0, |
| "epoch": 0.8954918032786885, |
| "grad_norm": 71.76630401611328, |
| "learning_rate": 1.0417332325579518e-06, |
| "logp/chosen": -772.0, |
| "logp/rejected": -820.0, |
| "loss": 0.94970703125, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 2.8125, |
| "reward/rejected": -3.40625, |
| "step": 874 |
| }, |
| { |
| "approx. KL/chosen": 12.3125, |
| "approx. KL/rejected": 274.0, |
| "epoch": 0.8965163934426229, |
| "grad_norm": 17.422889709472656, |
| "learning_rate": 1.037114140989945e-06, |
| "logp/chosen": -784.0, |
| "logp/rejected": -1184.0, |
| "loss": 0.76611328125, |
| "nll_loss": 0.64453125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.625, |
| "reward/margin": 4.40625, |
| "reward/rejected": -3.78125, |
| "step": 875 |
| }, |
| { |
| "approx. KL/chosen": 7.0, |
| "approx. KL/rejected": 213.0, |
| "epoch": 0.8975409836065574, |
| "grad_norm": 36.77158737182617, |
| "learning_rate": 1.0325381043013546e-06, |
| "logp/chosen": -832.0, |
| "logp/rejected": -1512.0, |
| "loss": 0.8193359375, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.30078125, |
| "reward/margin": 3.609375, |
| "reward/rejected": -3.296875, |
| "step": 876 |
| }, |
| { |
| "approx. KL/chosen": 66.0, |
| "approx. KL/rejected": 251.0, |
| "epoch": 0.8985655737704918, |
| "grad_norm": 115.37538146972656, |
| "learning_rate": 1.0280051810789742e-06, |
| "logp/chosen": -780.0, |
| "logp/rejected": -916.0, |
| "loss": 1.72607421875, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.0, |
| "reward/margin": 2.046875, |
| "reward/rejected": -3.046875, |
| "step": 877 |
| }, |
| { |
| "approx. KL/chosen": 9.75, |
| "approx. KL/rejected": 81.5, |
| "epoch": 0.8995901639344263, |
| "grad_norm": 60.94810104370117, |
| "learning_rate": 1.0235154293576224e-06, |
| "logp/chosen": -468.0, |
| "logp/rejected": -548.0, |
| "loss": 1.46826171875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.5625, |
| "reward/margin": 0.7734375, |
| "reward/rejected": -1.3359375, |
| "step": 878 |
| }, |
| { |
| "approx. KL/chosen": 12.25, |
| "approx. KL/rejected": 118.0, |
| "epoch": 0.9006147540983607, |
| "grad_norm": 86.2357177734375, |
| "learning_rate": 1.0190689066193911e-06, |
| "logp/chosen": -720.0, |
| "logp/rejected": -1072.0, |
| "loss": 1.36279296875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 1.6484375, |
| "reward/rejected": -1.8046875, |
| "step": 879 |
| }, |
| { |
| "approx. KL/chosen": 47.25, |
| "approx. KL/rejected": 157.0, |
| "epoch": 0.9016393442622951, |
| "grad_norm": 105.29753875732422, |
| "learning_rate": 1.014665669792915e-06, |
| "logp/chosen": -1048.0, |
| "logp/rejected": -952.0, |
| "loss": 1.51611328125, |
| "nll_loss": 0.490234375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 1.296875, |
| "reward/rejected": -1.953125, |
| "step": 880 |
| }, |
| { |
| "approx. KL/chosen": 40.0, |
| "approx. KL/rejected": 57.5, |
| "epoch": 0.9026639344262295, |
| "grad_norm": 91.34139251708984, |
| "learning_rate": 1.010305775252642e-06, |
| "logp/chosen": -864.0, |
| "logp/rejected": -888.0, |
| "loss": 1.19140625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.0, |
| "reward/margin": 1.625, |
| "reward/rejected": -1.625, |
| "step": 881 |
| }, |
| { |
| "approx. KL/chosen": 69.0, |
| "approx. KL/rejected": 249.0, |
| "epoch": 0.9036885245901639, |
| "grad_norm": 95.68619537353516, |
| "learning_rate": 1.0059892788181083e-06, |
| "logp/chosen": -880.0, |
| "logp/rejected": -1512.0, |
| "loss": 1.54638671875, |
| "nll_loss": 0.66015625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.25, |
| "reward/margin": 2.4375, |
| "reward/rejected": -3.6875, |
| "step": 882 |
| }, |
| { |
| "approx. KL/chosen": 54.5, |
| "approx. KL/rejected": 28.0, |
| "epoch": 0.9047131147540983, |
| "grad_norm": 118.37432861328125, |
| "learning_rate": 1.0017162357532293e-06, |
| "logp/chosen": -1024.0, |
| "logp/rejected": -1152.0, |
| "loss": 1.875, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -1.6015625, |
| "reward/margin": -0.6015625, |
| "reward/rejected": -1.0, |
| "step": 883 |
| }, |
| { |
| "approx. KL/chosen": 54.0, |
| "approx. KL/rejected": 474.0, |
| "epoch": 0.9057377049180327, |
| "grad_norm": 54.488346099853516, |
| "learning_rate": 9.974867007655857e-07, |
| "logp/chosen": -1000.0, |
| "logp/rejected": -1680.0, |
| "loss": 0.7783203125, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.201171875, |
| "reward/margin": 5.40625, |
| "reward/rejected": -5.59375, |
| "step": 884 |
| }, |
| { |
| "approx. KL/chosen": 10.3125, |
| "approx. KL/rejected": 57.25, |
| "epoch": 0.9067622950819673, |
| "grad_norm": 85.2398452758789, |
| "learning_rate": 9.933007280057282e-07, |
| "logp/chosen": -884.0, |
| "logp/rejected": -928.0, |
| "loss": 1.4560546875, |
| "nll_loss": 0.625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.474609375, |
| "reward/margin": 0.4765625, |
| "reward/rejected": -0.953125, |
| "step": 885 |
| }, |
| { |
| "approx. KL/chosen": 46.0, |
| "approx. KL/rejected": 230.0, |
| "epoch": 0.9077868852459017, |
| "grad_norm": 78.31346130371094, |
| "learning_rate": 9.8915837106648e-07, |
| "logp/chosen": -828.0, |
| "logp/rejected": -1000.0, |
| "loss": 1.186279296875, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.376953125, |
| "reward/margin": 2.90625, |
| "reward/rejected": -3.28125, |
| "step": 886 |
| }, |
| { |
| "approx. KL/chosen": 14.1875, |
| "approx. KL/rejected": 66.0, |
| "epoch": 0.9088114754098361, |
| "grad_norm": 68.11192321777344, |
| "learning_rate": 9.850596829822569e-07, |
| "logp/chosen": -472.0, |
| "logp/rejected": -740.0, |
| "loss": 1.10107421875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.275390625, |
| "reward/margin": 1.375, |
| "reward/rejected": -1.65625, |
| "step": 887 |
| }, |
| { |
| "approx. KL/chosen": 39.5, |
| "approx. KL/rejected": 61.0, |
| "epoch": 0.9098360655737705, |
| "grad_norm": 62.523746490478516, |
| "learning_rate": 9.810047162283793e-07, |
| "logp/chosen": -760.0, |
| "logp/rejected": -676.0, |
| "loss": 1.07861328125, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 1.3984375, |
| "reward/rejected": -1.703125, |
| "step": 888 |
| }, |
| { |
| "approx. KL/chosen": 34.25, |
| "approx. KL/rejected": 414.0, |
| "epoch": 0.9108606557377049, |
| "grad_norm": 58.56706237792969, |
| "learning_rate": 9.769935227204097e-07, |
| "logp/chosen": -836.0, |
| "logp/rejected": -1032.0, |
| "loss": 1.15087890625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.1171875, |
| "reward/margin": 3.015625, |
| "reward/rejected": -4.125, |
| "step": 889 |
| }, |
| { |
| "approx. KL/chosen": 61.25, |
| "approx. KL/rejected": 241.0, |
| "epoch": 0.9118852459016393, |
| "grad_norm": 109.90747833251953, |
| "learning_rate": 9.730261538134804e-07, |
| "logp/chosen": -1544.0, |
| "logp/rejected": -1648.0, |
| "loss": 1.26123046875, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 3.359375, |
| "reward/rejected": -3.90625, |
| "step": 890 |
| }, |
| { |
| "approx. KL/chosen": 50.0, |
| "approx. KL/rejected": 80.0, |
| "epoch": 0.9129098360655737, |
| "grad_norm": 116.75489044189453, |
| "learning_rate": 9.691026603016416e-07, |
| "logp/chosen": -1352.0, |
| "logp/rejected": -1296.0, |
| "loss": 1.66845703125, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 0.71484375, |
| "reward/rejected": -1.3203125, |
| "step": 891 |
| }, |
| { |
| "approx. KL/chosen": 49.25, |
| "approx. KL/rejected": 200.0, |
| "epoch": 0.9139344262295082, |
| "grad_norm": 81.30648803710938, |
| "learning_rate": 9.652230924172071e-07, |
| "logp/chosen": -1020.0, |
| "logp/rejected": -1144.0, |
| "loss": 1.25634765625, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.86328125, |
| "reward/margin": 1.8828125, |
| "reward/rejected": -2.734375, |
| "step": 892 |
| }, |
| { |
| "approx. KL/chosen": 51.0, |
| "approx. KL/rejected": 152.0, |
| "epoch": 0.9149590163934426, |
| "grad_norm": 54.2677001953125, |
| "learning_rate": 9.613874998301146e-07, |
| "logp/chosen": -1480.0, |
| "logp/rejected": -1720.0, |
| "loss": 0.9521484375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.703125, |
| "reward/margin": 3.078125, |
| "reward/rejected": -2.375, |
| "step": 893 |
| }, |
| { |
| "approx. KL/chosen": 25.375, |
| "approx. KL/rejected": 143.0, |
| "epoch": 0.9159836065573771, |
| "grad_norm": 55.657249450683594, |
| "learning_rate": 9.575959316472853e-07, |
| "logp/chosen": -956.0, |
| "logp/rejected": -1280.0, |
| "loss": 0.896484375, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.62109375, |
| "reward/margin": 2.765625, |
| "reward/rejected": -2.140625, |
| "step": 894 |
| }, |
| { |
| "approx. KL/chosen": 45.5, |
| "approx. KL/rejected": 242.0, |
| "epoch": 0.9170081967213115, |
| "grad_norm": 26.736234664916992, |
| "learning_rate": 9.538484364120005e-07, |
| "logp/chosen": -1008.0, |
| "logp/rejected": -924.0, |
| "loss": 0.74609375, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -0.00048828125, |
| "reward/margin": 3.203125, |
| "reward/rejected": -3.203125, |
| "step": 895 |
| }, |
| { |
| "approx. KL/chosen": 65.0, |
| "approx. KL/rejected": 37.0, |
| "epoch": 0.9180327868852459, |
| "grad_norm": 107.76377868652344, |
| "learning_rate": 9.501450621032765e-07, |
| "logp/chosen": -1232.0, |
| "logp/rejected": -1232.0, |
| "loss": 1.4736328125, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.5, |
| "reward/margin": 0.59765625, |
| "reward/rejected": -1.1015625, |
| "step": 896 |
| }, |
| { |
| "approx. KL/chosen": 35.25, |
| "approx. KL/rejected": 135.0, |
| "epoch": 0.9190573770491803, |
| "grad_norm": 95.2989730834961, |
| "learning_rate": 9.464858561352512e-07, |
| "logp/chosen": -900.0, |
| "logp/rejected": -960.0, |
| "loss": 1.2548828125, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.75, |
| "reward/margin": 1.5703125, |
| "reward/rejected": -2.328125, |
| "step": 897 |
| }, |
| { |
| "approx. KL/chosen": 6.3125, |
| "approx. KL/rejected": 45.75, |
| "epoch": 0.9200819672131147, |
| "grad_norm": 43.79899597167969, |
| "learning_rate": 9.428708653565771e-07, |
| "logp/chosen": -446.0, |
| "logp/rejected": -442.0, |
| "loss": 0.94775390625, |
| "nll_loss": 0.453125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.02490234375, |
| "reward/margin": 1.046875, |
| "reward/rejected": -1.0234375, |
| "step": 898 |
| }, |
| { |
| "approx. KL/chosen": 62.0, |
| "approx. KL/rejected": 223.0, |
| "epoch": 0.9211065573770492, |
| "grad_norm": 97.92496490478516, |
| "learning_rate": 9.39300136049822e-07, |
| "logp/chosen": -920.0, |
| "logp/rejected": -864.0, |
| "loss": 1.412109375, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.0240478515625, |
| "reward/margin": 3.15625, |
| "reward/rejected": -3.125, |
| "step": 899 |
| }, |
| { |
| "approx. KL/chosen": 7.5, |
| "approx. KL/rejected": 104.5, |
| "epoch": 0.9221311475409836, |
| "grad_norm": 38.808006286621094, |
| "learning_rate": 9.357737139308758e-07, |
| "logp/chosen": -688.0, |
| "logp/rejected": -928.0, |
| "loss": 0.81396484375, |
| "nll_loss": 0.53515625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.10009765625, |
| "reward/margin": 2.078125, |
| "reward/rejected": -2.171875, |
| "step": 900 |
| }, |
| { |
| "approx. KL/chosen": 40.75, |
| "approx. KL/rejected": 91.5, |
| "epoch": 0.923155737704918, |
| "grad_norm": 59.0588264465332, |
| "learning_rate": 9.322916441483656e-07, |
| "logp/chosen": -696.0, |
| "logp/rejected": -844.0, |
| "loss": 1.36279296875, |
| "nll_loss": 0.59765625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.82421875, |
| "reward/margin": 0.55078125, |
| "reward/rejected": -1.375, |
| "step": 901 |
| }, |
| { |
| "approx. KL/chosen": 11.0, |
| "approx. KL/rejected": 328.0, |
| "epoch": 0.9241803278688525, |
| "grad_norm": 16.29414939880371, |
| "learning_rate": 9.288539712830772e-07, |
| "logp/chosen": -984.0, |
| "logp/rejected": -1504.0, |
| "loss": 0.71923828125, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 1.0, |
| "reward/chosen": 0.30078125, |
| "reward/margin": 4.625, |
| "reward/rejected": -4.3125, |
| "step": 902 |
| }, |
| { |
| "approx. KL/chosen": 15.75, |
| "approx. KL/rejected": 166.0, |
| "epoch": 0.9252049180327869, |
| "grad_norm": 41.79083251953125, |
| "learning_rate": 9.254607393473844e-07, |
| "logp/chosen": -812.0, |
| "logp/rejected": -852.0, |
| "loss": 0.792724609375, |
| "nll_loss": 0.59375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.0498046875, |
| "reward/margin": 3.109375, |
| "reward/rejected": -3.171875, |
| "step": 903 |
| }, |
| { |
| "approx. KL/chosen": 30.25, |
| "approx. KL/rejected": 406.0, |
| "epoch": 0.9262295081967213, |
| "grad_norm": 38.241249084472656, |
| "learning_rate": 9.221119917846866e-07, |
| "logp/chosen": -932.0, |
| "logp/rejected": -1664.0, |
| "loss": 0.81884765625, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.050048828125, |
| "reward/margin": 4.71875, |
| "reward/rejected": -4.65625, |
| "step": 904 |
| }, |
| { |
| "approx. KL/chosen": 26.625, |
| "approx. KL/rejected": 126.5, |
| "epoch": 0.9272540983606558, |
| "grad_norm": 88.80674743652344, |
| "learning_rate": 9.188077714688514e-07, |
| "logp/chosen": -860.0, |
| "logp/rejected": -940.0, |
| "loss": 1.518798828125, |
| "nll_loss": 0.50390625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 1.5, |
| "reward/rejected": -1.703125, |
| "step": 905 |
| }, |
| { |
| "approx. KL/chosen": 32.0, |
| "approx. KL/rejected": 486.0, |
| "epoch": 0.9282786885245902, |
| "grad_norm": 83.42424011230469, |
| "learning_rate": 9.15548120703665e-07, |
| "logp/chosen": -696.0, |
| "logp/rejected": -1296.0, |
| "loss": 1.0400390625, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.201171875, |
| "reward/margin": 3.65625, |
| "reward/rejected": -3.859375, |
| "step": 906 |
| }, |
| { |
| "approx. KL/chosen": 3.890625, |
| "approx. KL/rejected": 33.0, |
| "epoch": 0.9293032786885246, |
| "grad_norm": 115.71084594726562, |
| "learning_rate": 9.123330812222931e-07, |
| "logp/chosen": -1176.0, |
| "logp/rejected": -796.0, |
| "loss": 1.669921875, |
| "nll_loss": 0.61328125, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -0.2119140625, |
| "reward/margin": -0.138671875, |
| "reward/rejected": -0.07470703125, |
| "step": 907 |
| }, |
| { |
| "approx. KL/chosen": 24.75, |
| "approx. KL/rejected": 71.0, |
| "epoch": 0.930327868852459, |
| "grad_norm": 65.53607940673828, |
| "learning_rate": 9.091626941867442e-07, |
| "logp/chosen": -728.0, |
| "logp/rejected": -656.0, |
| "loss": 1.1484375, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.77734375, |
| "reward/margin": 1.0546875, |
| "reward/rejected": -1.828125, |
| "step": 908 |
| }, |
| { |
| "approx. KL/chosen": 72.0, |
| "approx. KL/rejected": 254.0, |
| "epoch": 0.9313524590163934, |
| "grad_norm": 133.44232177734375, |
| "learning_rate": 9.060370001873442e-07, |
| "logp/chosen": -1160.0, |
| "logp/rejected": -1456.0, |
| "loss": 1.87060546875, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.6015625, |
| "reward/margin": 1.1953125, |
| "reward/rejected": -2.796875, |
| "step": 909 |
| }, |
| { |
| "approx. KL/chosen": 12.25, |
| "approx. KL/rejected": 76.0, |
| "epoch": 0.9323770491803278, |
| "grad_norm": 84.35095977783203, |
| "learning_rate": 9.029560392422149e-07, |
| "logp/chosen": -864.0, |
| "logp/rejected": -1072.0, |
| "loss": 0.977294921875, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 1.203125, |
| "reward/rejected": -1.75, |
| "step": 910 |
| }, |
| { |
| "approx. KL/chosen": 14.0, |
| "approx. KL/rejected": 60.25, |
| "epoch": 0.9334016393442623, |
| "grad_norm": 95.01129913330078, |
| "learning_rate": 8.999198507967641e-07, |
| "logp/chosen": -856.0, |
| "logp/rejected": -924.0, |
| "loss": 1.36669921875, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 0.7734375, |
| "reward/rejected": -0.9765625, |
| "step": 911 |
| }, |
| { |
| "approx. KL/chosen": 38.5, |
| "approx. KL/rejected": 146.0, |
| "epoch": 0.9344262295081968, |
| "grad_norm": 42.65457534790039, |
| "learning_rate": 8.969284737231782e-07, |
| "logp/chosen": -752.0, |
| "logp/rejected": -816.0, |
| "loss": 0.89208984375, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.1005859375, |
| "reward/margin": 2.296875, |
| "reward/rejected": -2.40625, |
| "step": 912 |
| }, |
| { |
| "approx. KL/chosen": 12.3125, |
| "approx. KL/rejected": 81.0, |
| "epoch": 0.9354508196721312, |
| "grad_norm": 53.09586715698242, |
| "learning_rate": 8.939819463199267e-07, |
| "logp/chosen": -840.0, |
| "logp/rejected": -884.0, |
| "loss": 0.907470703125, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.02490234375, |
| "reward/margin": 1.9609375, |
| "reward/rejected": -1.984375, |
| "step": 913 |
| }, |
| { |
| "approx. KL/chosen": 29.0, |
| "approx. KL/rejected": 232.0, |
| "epoch": 0.9364754098360656, |
| "grad_norm": 122.04402923583984, |
| "learning_rate": 8.910803063112691e-07, |
| "logp/chosen": -1128.0, |
| "logp/rejected": -1256.0, |
| "loss": 1.24755859375, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.67578125, |
| "reward/margin": 2.46875, |
| "reward/rejected": -3.140625, |
| "step": 914 |
| }, |
| { |
| "approx. KL/chosen": 88.0, |
| "approx. KL/rejected": 96.0, |
| "epoch": 0.9375, |
| "grad_norm": 125.13658142089844, |
| "learning_rate": 8.882235908467751e-07, |
| "logp/chosen": -1168.0, |
| "logp/rejected": -1016.0, |
| "loss": 1.85498046875, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.703125, |
| "reward/margin": 0.1748046875, |
| "reward/rejected": -1.875, |
| "step": 915 |
| }, |
| { |
| "approx. KL/chosen": 9.0, |
| "approx. KL/rejected": 89.5, |
| "epoch": 0.9385245901639344, |
| "grad_norm": 112.24314880371094, |
| "learning_rate": 8.854118365008466e-07, |
| "logp/chosen": -1216.0, |
| "logp/rejected": -1696.0, |
| "loss": 1.26318359375, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.25, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 0.77734375, |
| "reward/rejected": -1.078125, |
| "step": 916 |
| }, |
| { |
| "approx. KL/chosen": 99.5, |
| "approx. KL/rejected": 167.0, |
| "epoch": 0.9395491803278688, |
| "grad_norm": 110.44268798828125, |
| "learning_rate": 8.826450792722511e-07, |
| "logp/chosen": -1240.0, |
| "logp/rejected": -1488.0, |
| "loss": 1.105224609375, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.40234375, |
| "reward/margin": 3.5625, |
| "reward/rejected": -3.15625, |
| "step": 917 |
| }, |
| { |
| "approx. KL/chosen": 13.25, |
| "approx. KL/rejected": 91.0, |
| "epoch": 0.9405737704918032, |
| "grad_norm": 69.96798706054688, |
| "learning_rate": 8.799233545836584e-07, |
| "logp/chosen": -792.0, |
| "logp/rejected": -1000.0, |
| "loss": 0.922119140625, |
| "nll_loss": 0.51171875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 1.25, |
| "reward/rejected": -2.109375, |
| "step": 918 |
| }, |
| { |
| "approx. KL/chosen": 10.5, |
| "approx. KL/rejected": 220.0, |
| "epoch": 0.9415983606557377, |
| "grad_norm": 27.330795288085938, |
| "learning_rate": 8.772466972811913e-07, |
| "logp/chosen": -724.0, |
| "logp/rejected": -1112.0, |
| "loss": 0.853515625, |
| "nll_loss": 0.6171875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 2.90625, |
| "reward/rejected": -3.203125, |
| "step": 919 |
| }, |
| { |
| "approx. KL/chosen": 19.0, |
| "approx. KL/rejected": 133.0, |
| "epoch": 0.9426229508196722, |
| "grad_norm": 38.076011657714844, |
| "learning_rate": 8.74615141633973e-07, |
| "logp/chosen": -1176.0, |
| "logp/rejected": -700.0, |
| "loss": 0.97900390625, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.275390625, |
| "reward/margin": 2.328125, |
| "reward/rejected": -2.609375, |
| "step": 920 |
| }, |
| { |
| "approx. KL/chosen": 19.25, |
| "approx. KL/rejected": 119.0, |
| "epoch": 0.9436475409836066, |
| "grad_norm": 48.588199615478516, |
| "learning_rate": 8.720287213336966e-07, |
| "logp/chosen": -784.0, |
| "logp/rejected": -968.0, |
| "loss": 0.883056640625, |
| "nll_loss": 0.48828125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.953125, |
| "reward/margin": 1.5390625, |
| "reward/rejected": -2.484375, |
| "step": 921 |
| }, |
| { |
| "approx. KL/chosen": 17.0, |
| "approx. KL/rejected": 154.0, |
| "epoch": 0.944672131147541, |
| "grad_norm": 86.11721801757812, |
| "learning_rate": 8.694874694941863e-07, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1400.0, |
| "loss": 1.09716796875, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 1.5, |
| "reward/rejected": -2.40625, |
| "step": 922 |
| }, |
| { |
| "approx. KL/chosen": 51.25, |
| "approx. KL/rejected": 74.5, |
| "epoch": 0.9456967213114754, |
| "grad_norm": 88.82428741455078, |
| "learning_rate": 8.669914186509774e-07, |
| "logp/chosen": -1088.0, |
| "logp/rejected": -712.0, |
| "loss": 1.49560546875, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 1.0546875, |
| "reward/rejected": -1.703125, |
| "step": 923 |
| }, |
| { |
| "approx. KL/chosen": 12.125, |
| "approx. KL/rejected": 79.0, |
| "epoch": 0.9467213114754098, |
| "grad_norm": 22.27496910095215, |
| "learning_rate": 8.645406007608986e-07, |
| "logp/chosen": -498.0, |
| "logp/rejected": -532.0, |
| "loss": 0.79052734375, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.1689453125, |
| "reward/margin": 1.859375, |
| "reward/rejected": -2.015625, |
| "step": 924 |
| }, |
| { |
| "approx. KL/chosen": 43.25, |
| "approx. KL/rejected": 41.5, |
| "epoch": 0.9477459016393442, |
| "grad_norm": 134.7558135986328, |
| "learning_rate": 8.62135047201664e-07, |
| "logp/chosen": -816.0, |
| "logp/rejected": -1040.0, |
| "loss": 1.9541015625, |
| "nll_loss": 0.54296875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.94921875, |
| "reward/margin": -0.42578125, |
| "reward/rejected": -0.5234375, |
| "step": 925 |
| }, |
| { |
| "approx. KL/chosen": 29.5, |
| "approx. KL/rejected": 276.0, |
| "epoch": 0.9487704918032787, |
| "grad_norm": 69.66436004638672, |
| "learning_rate": 8.597747887714689e-07, |
| "logp/chosen": -668.0, |
| "logp/rejected": -1248.0, |
| "loss": 1.291015625, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 2.84375, |
| "reward/rejected": -3.65625, |
| "step": 926 |
| }, |
| { |
| "approx. KL/chosen": 22.875, |
| "approx. KL/rejected": 96.5, |
| "epoch": 0.9497950819672131, |
| "grad_norm": 98.17277526855469, |
| "learning_rate": 8.574598556885987e-07, |
| "logp/chosen": -1312.0, |
| "logp/rejected": -1216.0, |
| "loss": 1.339599609375, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.23828125, |
| "reward/margin": 1.7578125, |
| "reward/rejected": -1.9921875, |
| "step": 927 |
| }, |
| { |
| "approx. KL/chosen": 17.5, |
| "approx. KL/rejected": 188.0, |
| "epoch": 0.9508196721311475, |
| "grad_norm": 79.86397552490234, |
| "learning_rate": 8.551902775910388e-07, |
| "logp/chosen": -852.0, |
| "logp/rejected": -1360.0, |
| "loss": 1.20068359375, |
| "nll_loss": 0.609375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.9765625, |
| "reward/margin": 2.0, |
| "reward/rejected": -2.96875, |
| "step": 928 |
| }, |
| { |
| "approx. KL/chosen": 31.25, |
| "approx. KL/rejected": 234.0, |
| "epoch": 0.951844262295082, |
| "grad_norm": 52.33475112915039, |
| "learning_rate": 8.529660835360981e-07, |
| "logp/chosen": -760.0, |
| "logp/rejected": -1416.0, |
| "loss": 1.03662109375, |
| "nll_loss": 0.5546875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.251953125, |
| "reward/margin": 3.453125, |
| "reward/rejected": -3.703125, |
| "step": 929 |
| }, |
| { |
| "approx. KL/chosen": 134.0, |
| "approx. KL/rejected": 249.0, |
| "epoch": 0.9528688524590164, |
| "grad_norm": 121.5876235961914, |
| "learning_rate": 8.507873020000355e-07, |
| "logp/chosen": -968.0, |
| "logp/rejected": -1048.0, |
| "loss": 2.759765625, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -2.5625, |
| "reward/margin": 0.34765625, |
| "reward/rejected": -2.90625, |
| "step": 930 |
| }, |
| { |
| "approx. KL/chosen": 19.375, |
| "approx. KL/rejected": 282.0, |
| "epoch": 0.9538934426229508, |
| "grad_norm": 55.0185546875, |
| "learning_rate": 8.486539608776944e-07, |
| "logp/chosen": -748.0, |
| "logp/rejected": -1472.0, |
| "loss": 0.81494140625, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.23828125, |
| "reward/margin": 3.859375, |
| "reward/rejected": -3.625, |
| "step": 931 |
| }, |
| { |
| "approx. KL/chosen": 40.0, |
| "approx. KL/rejected": 60.5, |
| "epoch": 0.9549180327868853, |
| "grad_norm": 87.49356842041016, |
| "learning_rate": 8.465660874821469e-07, |
| "logp/chosen": -1480.0, |
| "logp/rejected": -1536.0, |
| "loss": 1.19580078125, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.400390625, |
| "reward/margin": 1.1953125, |
| "reward/rejected": -1.59375, |
| "step": 932 |
| }, |
| { |
| "approx. KL/chosen": 32.25, |
| "approx. KL/rejected": 59.0, |
| "epoch": 0.9559426229508197, |
| "grad_norm": 110.7561264038086, |
| "learning_rate": 8.445237085443443e-07, |
| "logp/chosen": -1136.0, |
| "logp/rejected": -1216.0, |
| "loss": 1.19970703125, |
| "nll_loss": 0.4765625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -0.8515625, |
| "reward/margin": 0.451171875, |
| "reward/rejected": -1.3046875, |
| "step": 933 |
| }, |
| { |
| "approx. KL/chosen": 44.25, |
| "approx. KL/rejected": 249.0, |
| "epoch": 0.9569672131147541, |
| "grad_norm": 75.0616683959961, |
| "learning_rate": 8.425268502127742e-07, |
| "logp/chosen": -800.0, |
| "logp/rejected": -1904.0, |
| "loss": 0.7978515625, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.150390625, |
| "reward/margin": 3.75, |
| "reward/rejected": -3.90625, |
| "step": 934 |
| }, |
| { |
| "approx. KL/chosen": 16.25, |
| "approx. KL/rejected": 272.0, |
| "epoch": 0.9579918032786885, |
| "grad_norm": 69.26927185058594, |
| "learning_rate": 8.405755380531256e-07, |
| "logp/chosen": -960.0, |
| "logp/rejected": -1656.0, |
| "loss": 0.9619140625, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.451171875, |
| "reward/margin": 3.203125, |
| "reward/rejected": -3.65625, |
| "step": 935 |
| }, |
| { |
| "approx. KL/chosen": 46.0, |
| "approx. KL/rejected": 223.0, |
| "epoch": 0.9590163934426229, |
| "grad_norm": 68.74922943115234, |
| "learning_rate": 8.386697970479629e-07, |
| "logp/chosen": -980.0, |
| "logp/rejected": -1312.0, |
| "loss": 0.989990234375, |
| "nll_loss": 0.546875, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.201171875, |
| "reward/margin": 3.421875, |
| "reward/rejected": -3.625, |
| "step": 936 |
| }, |
| { |
| "approx. KL/chosen": 18.25, |
| "approx. KL/rejected": 180.0, |
| "epoch": 0.9600409836065574, |
| "grad_norm": 39.22175216674805, |
| "learning_rate": 8.368096515964026e-07, |
| "logp/chosen": -668.0, |
| "logp/rejected": -1064.0, |
| "loss": 0.81591796875, |
| "nll_loss": 0.498046875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.349609375, |
| "reward/margin": 3.0625, |
| "reward/rejected": -2.703125, |
| "step": 937 |
| }, |
| { |
| "approx. KL/chosen": 50.0, |
| "approx. KL/rejected": 264.0, |
| "epoch": 0.9610655737704918, |
| "grad_norm": 25.548437118530273, |
| "learning_rate": 8.349951255138063e-07, |
| "logp/chosen": -792.0, |
| "logp/rejected": -1328.0, |
| "loss": 0.66357421875, |
| "nll_loss": 0.515625, |
| "reward/accuracy": 1.0, |
| "reward/chosen": -1.0234375, |
| "reward/margin": 3.03125, |
| "reward/rejected": -4.0625, |
| "step": 938 |
| }, |
| { |
| "approx. KL/chosen": 26.5, |
| "approx. KL/rejected": 98.0, |
| "epoch": 0.9620901639344263, |
| "grad_norm": 83.47401428222656, |
| "learning_rate": 8.332262420314711e-07, |
| "logp/chosen": -964.0, |
| "logp/rejected": -1360.0, |
| "loss": 1.1279296875, |
| "nll_loss": 0.5390625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 1.6484375, |
| "reward/rejected": -1.8515625, |
| "step": 939 |
| }, |
| { |
| "approx. KL/chosen": 41.75, |
| "approx. KL/rejected": 251.0, |
| "epoch": 0.9631147540983607, |
| "grad_norm": 56.641273498535156, |
| "learning_rate": 8.315030237963343e-07, |
| "logp/chosen": -576.0, |
| "logp/rejected": -952.0, |
| "loss": 1.068359375, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -1.078125, |
| "reward/margin": 1.828125, |
| "reward/rejected": -2.90625, |
| "step": 940 |
| }, |
| { |
| "approx. KL/chosen": 63.0, |
| "approx. KL/rejected": 105.5, |
| "epoch": 0.9641393442622951, |
| "grad_norm": 133.4600830078125, |
| "learning_rate": 8.298254928706822e-07, |
| "logp/chosen": -1304.0, |
| "logp/rejected": -1504.0, |
| "loss": 1.68408203125, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": -1.9765625, |
| "reward/margin": 0.1953125, |
| "reward/rejected": -2.171875, |
| "step": 941 |
| }, |
| { |
| "approx. KL/chosen": 12.5, |
| "approx. KL/rejected": 50.25, |
| "epoch": 0.9651639344262295, |
| "grad_norm": 82.82538604736328, |
| "learning_rate": 8.281936707318701e-07, |
| "logp/chosen": -720.0, |
| "logp/rejected": -708.0, |
| "loss": 1.2763671875, |
| "nll_loss": 0.486328125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.400390625, |
| "reward/margin": 0.92578125, |
| "reward/rejected": -0.5234375, |
| "step": 942 |
| }, |
| { |
| "approx. KL/chosen": 23.25, |
| "approx. KL/rejected": 86.5, |
| "epoch": 0.9661885245901639, |
| "grad_norm": 72.79498291015625, |
| "learning_rate": 8.266075782720448e-07, |
| "logp/chosen": -736.0, |
| "logp/rejected": -920.0, |
| "loss": 1.51611328125, |
| "nll_loss": 0.64453125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.44921875, |
| "reward/margin": 1.140625, |
| "reward/rejected": -1.59375, |
| "step": 943 |
| }, |
| { |
| "approx. KL/chosen": 104.0, |
| "approx. KL/rejected": 512.0, |
| "epoch": 0.9672131147540983, |
| "grad_norm": 107.07550048828125, |
| "learning_rate": 8.250672357978779e-07, |
| "logp/chosen": -1112.0, |
| "logp/rejected": -1880.0, |
| "loss": 1.30322265625, |
| "nll_loss": 0.55859375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.75, |
| "reward/margin": 4.3125, |
| "reward/rejected": -5.0625, |
| "step": 944 |
| }, |
| { |
| "approx. KL/chosen": 42.25, |
| "approx. KL/rejected": 210.0, |
| "epoch": 0.9682377049180327, |
| "grad_norm": 97.028076171875, |
| "learning_rate": 8.235726630303068e-07, |
| "logp/chosen": -1024.0, |
| "logp/rejected": -1344.0, |
| "loss": 1.5107421875, |
| "nll_loss": 0.5859375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.44921875, |
| "reward/margin": 2.703125, |
| "reward/rejected": -2.25, |
| "step": 945 |
| }, |
| { |
| "approx. KL/chosen": 48.5, |
| "approx. KL/rejected": 124.0, |
| "epoch": 0.9692622950819673, |
| "grad_norm": 84.42829132080078, |
| "learning_rate": 8.221238791042801e-07, |
| "logp/chosen": -556.0, |
| "logp/rejected": -568.0, |
| "loss": 1.44091796875, |
| "nll_loss": 0.48828125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.6953125, |
| "reward/margin": 1.5703125, |
| "reward/rejected": -2.265625, |
| "step": 946 |
| }, |
| { |
| "approx. KL/chosen": 42.0, |
| "approx. KL/rejected": 73.5, |
| "epoch": 0.9702868852459017, |
| "grad_norm": 72.13382720947266, |
| "learning_rate": 8.207209025685146e-07, |
| "logp/chosen": -800.0, |
| "logp/rejected": -940.0, |
| "loss": 1.1494140625, |
| "nll_loss": 0.5, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 1.0234375, |
| "reward/rejected": -1.625, |
| "step": 947 |
| }, |
| { |
| "approx. KL/chosen": 39.0, |
| "approx. KL/rejected": 161.0, |
| "epoch": 0.9713114754098361, |
| "grad_norm": 42.607696533203125, |
| "learning_rate": 8.193637513852574e-07, |
| "logp/chosen": -860.0, |
| "logp/rejected": -1432.0, |
| "loss": 0.7158203125, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.703125, |
| "reward/margin": 3.546875, |
| "reward/rejected": -2.84375, |
| "step": 948 |
| }, |
| { |
| "approx. KL/chosen": 26.125, |
| "approx. KL/rejected": 173.0, |
| "epoch": 0.9723360655737705, |
| "grad_norm": 147.6056365966797, |
| "learning_rate": 8.18052442930055e-07, |
| "logp/chosen": -1176.0, |
| "logp/rejected": -1512.0, |
| "loss": 1.423095703125, |
| "nll_loss": 0.484375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.5625, |
| "reward/margin": 1.71875, |
| "reward/rejected": -2.28125, |
| "step": 949 |
| }, |
| { |
| "approx. KL/chosen": 44.0, |
| "approx. KL/rejected": 196.0, |
| "epoch": 0.9733606557377049, |
| "grad_norm": 86.05207824707031, |
| "learning_rate": 8.16786993991532e-07, |
| "logp/chosen": -872.0, |
| "logp/rejected": -1024.0, |
| "loss": 1.42431640625, |
| "nll_loss": 0.62890625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.42578125, |
| "reward/margin": 2.140625, |
| "reward/rejected": -2.5625, |
| "step": 950 |
| }, |
| { |
| "approx. KL/chosen": 113.0, |
| "approx. KL/rejected": 122.0, |
| "epoch": 0.9743852459016393, |
| "grad_norm": 146.65261840820312, |
| "learning_rate": 8.155674207711752e-07, |
| "logp/chosen": -1280.0, |
| "logp/rejected": -1280.0, |
| "loss": 1.9599609375, |
| "nll_loss": 0.6328125, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.5, |
| "reward/margin": 0.8984375, |
| "reward/rejected": -2.40625, |
| "step": 951 |
| }, |
| { |
| "approx. KL/chosen": 80.5, |
| "approx. KL/rejected": 114.0, |
| "epoch": 0.9754098360655737, |
| "grad_norm": 95.41036224365234, |
| "learning_rate": 8.14393738883127e-07, |
| "logp/chosen": -1104.0, |
| "logp/rejected": -1504.0, |
| "loss": 1.72998046875, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.90234375, |
| "reward/margin": 1.046875, |
| "reward/rejected": -1.953125, |
| "step": 952 |
| }, |
| { |
| "approx. KL/chosen": 44.25, |
| "approx. KL/rejected": 243.0, |
| "epoch": 0.9764344262295082, |
| "grad_norm": 63.15992736816406, |
| "learning_rate": 8.132659633539844e-07, |
| "logp/chosen": -732.0, |
| "logp/rejected": -860.0, |
| "loss": 0.8564453125, |
| "nll_loss": 0.51953125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.65234375, |
| "reward/margin": 2.96875, |
| "reward/rejected": -3.625, |
| "step": 953 |
| }, |
| { |
| "approx. KL/chosen": 45.25, |
| "approx. KL/rejected": 138.0, |
| "epoch": 0.9774590163934426, |
| "grad_norm": 72.65413665771484, |
| "learning_rate": 8.12184108622608e-07, |
| "logp/chosen": -952.0, |
| "logp/rejected": -1048.0, |
| "loss": 1.39306640625, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.1513671875, |
| "reward/margin": 1.875, |
| "reward/rejected": -2.03125, |
| "step": 954 |
| }, |
| { |
| "approx. KL/chosen": 55.75, |
| "approx. KL/rejected": 202.0, |
| "epoch": 0.9784836065573771, |
| "grad_norm": 143.5027313232422, |
| "learning_rate": 8.111481885399363e-07, |
| "logp/chosen": -972.0, |
| "logp/rejected": -1336.0, |
| "loss": 2.63916015625, |
| "nll_loss": 0.56640625, |
| "reward/accuracy": 0.375, |
| "reward/chosen": 0.1748046875, |
| "reward/margin": 0.025390625, |
| "reward/rejected": 0.1513671875, |
| "step": 955 |
| }, |
| { |
| "approx. KL/chosen": 13.0, |
| "approx. KL/rejected": 200.0, |
| "epoch": 0.9795081967213115, |
| "grad_norm": 57.1081657409668, |
| "learning_rate": 8.101582163688087e-07, |
| "logp/chosen": -964.0, |
| "logp/rejected": -1128.0, |
| "loss": 0.962890625, |
| "nll_loss": 0.62109375, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 2.75, |
| "reward/rejected": -3.03125, |
| "step": 956 |
| }, |
| { |
| "approx. KL/chosen": 9.625, |
| "approx. KL/rejected": 111.5, |
| "epoch": 0.9805327868852459, |
| "grad_norm": 61.017337799072266, |
| "learning_rate": 8.092142047837947e-07, |
| "logp/chosen": -908.0, |
| "logp/rejected": -712.0, |
| "loss": 1.26513671875, |
| "nll_loss": 0.5625, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.55078125, |
| "reward/margin": 1.3046875, |
| "reward/rejected": -1.859375, |
| "step": 957 |
| }, |
| { |
| "approx. KL/chosen": 32.25, |
| "approx. KL/rejected": 54.0, |
| "epoch": 0.9815573770491803, |
| "grad_norm": 60.23340606689453, |
| "learning_rate": 8.083161658710338e-07, |
| "logp/chosen": -752.0, |
| "logp/rejected": -752.0, |
| "loss": 1.2646484375, |
| "nll_loss": 0.63671875, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -1.0546875, |
| "reward/margin": 0.55078125, |
| "reward/rejected": -1.6015625, |
| "step": 958 |
| }, |
| { |
| "approx. KL/chosen": 9.75, |
| "approx. KL/rejected": 306.0, |
| "epoch": 0.9825819672131147, |
| "grad_norm": 55.82500076293945, |
| "learning_rate": 8.074641111280777e-07, |
| "logp/chosen": -624.0, |
| "logp/rejected": -1048.0, |
| "loss": 1.24365234375, |
| "nll_loss": 0.6015625, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.462890625, |
| "reward/margin": 2.359375, |
| "reward/rejected": -2.828125, |
| "step": 959 |
| }, |
| { |
| "approx. KL/chosen": 32.5, |
| "approx. KL/rejected": 164.0, |
| "epoch": 0.9836065573770492, |
| "grad_norm": 109.49954986572266, |
| "learning_rate": 8.066580514637465e-07, |
| "logp/chosen": -752.0, |
| "logp/rejected": -844.0, |
| "loss": 1.279052734375, |
| "nll_loss": 0.5234375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.050048828125, |
| "reward/margin": 1.8984375, |
| "reward/rejected": -1.9453125, |
| "step": 960 |
| }, |
| { |
| "approx. KL/chosen": 16.5, |
| "approx. KL/rejected": 153.0, |
| "epoch": 0.9846311475409836, |
| "grad_norm": 29.572662353515625, |
| "learning_rate": 8.05897997197986e-07, |
| "logp/chosen": -696.0, |
| "logp/rejected": -792.0, |
| "loss": 0.900390625, |
| "nll_loss": 0.55078125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.1748046875, |
| "reward/margin": 2.828125, |
| "reward/rejected": -2.65625, |
| "step": 961 |
| }, |
| { |
| "approx. KL/chosen": 35.0, |
| "approx. KL/rejected": 102.0, |
| "epoch": 0.985655737704918, |
| "grad_norm": 104.77497100830078, |
| "learning_rate": 8.051839580617379e-07, |
| "logp/chosen": -828.0, |
| "logp/rejected": -1040.0, |
| "loss": 1.54736328125, |
| "nll_loss": 0.5078125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.703125, |
| "reward/margin": 1.046875, |
| "reward/rejected": -1.75, |
| "step": 962 |
| }, |
| { |
| "approx. KL/chosen": 22.25, |
| "approx. KL/rejected": 206.0, |
| "epoch": 0.9866803278688525, |
| "grad_norm": 90.39132690429688, |
| "learning_rate": 8.045159431968135e-07, |
| "logp/chosen": -1120.0, |
| "logp/rejected": -1384.0, |
| "loss": 1.3681640625, |
| "nll_loss": 0.6484375, |
| "reward/accuracy": 0.5, |
| "reward/chosen": -0.75, |
| "reward/margin": 1.5546875, |
| "reward/rejected": -2.296875, |
| "step": 963 |
| }, |
| { |
| "approx. KL/chosen": 24.0, |
| "approx. KL/rejected": 133.0, |
| "epoch": 0.9877049180327869, |
| "grad_norm": 92.08318328857422, |
| "learning_rate": 8.038939611557777e-07, |
| "logp/chosen": -1200.0, |
| "logp/rejected": -1320.0, |
| "loss": 1.21826171875, |
| "nll_loss": 0.65234375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.80078125, |
| "reward/margin": 1.703125, |
| "reward/rejected": -2.5, |
| "step": 964 |
| }, |
| { |
| "approx. KL/chosen": 110.0, |
| "approx. KL/rejected": 81.0, |
| "epoch": 0.9887295081967213, |
| "grad_norm": 78.67958068847656, |
| "learning_rate": 8.033180199018391e-07, |
| "logp/chosen": -1272.0, |
| "logp/rejected": -1352.0, |
| "loss": 1.05810546875, |
| "nll_loss": 0.58203125, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.3984375, |
| "reward/margin": 1.3046875, |
| "reward/rejected": -0.90234375, |
| "step": 965 |
| }, |
| { |
| "approx. KL/chosen": 20.25, |
| "approx. KL/rejected": 187.0, |
| "epoch": 0.9897540983606558, |
| "grad_norm": 93.0257568359375, |
| "learning_rate": 8.027881268087482e-07, |
| "logp/chosen": -756.0, |
| "logp/rejected": -1760.0, |
| "loss": 1.1943359375, |
| "nll_loss": 0.53125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": 0.25, |
| "reward/margin": 3.0, |
| "reward/rejected": -2.75, |
| "step": 966 |
| }, |
| { |
| "approx. KL/chosen": 51.0, |
| "approx. KL/rejected": 258.0, |
| "epoch": 0.9907786885245902, |
| "grad_norm": 71.42436981201172, |
| "learning_rate": 8.023042886607032e-07, |
| "logp/chosen": -880.0, |
| "logp/rejected": -1248.0, |
| "loss": 0.92822265625, |
| "nll_loss": 0.58984375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 3.5, |
| "reward/rejected": -3.796875, |
| "step": 967 |
| }, |
| { |
| "approx. KL/chosen": 14.0, |
| "approx. KL/rejected": 336.0, |
| "epoch": 0.9918032786885246, |
| "grad_norm": 79.12540435791016, |
| "learning_rate": 8.018665116522623e-07, |
| "logp/chosen": -940.0, |
| "logp/rejected": -1312.0, |
| "loss": 1.060546875, |
| "nll_loss": 0.5703125, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.2001953125, |
| "reward/margin": 3.515625, |
| "reward/rejected": -3.703125, |
| "step": 968 |
| }, |
| { |
| "approx. KL/chosen": 16.0, |
| "approx. KL/rejected": 476.0, |
| "epoch": 0.992827868852459, |
| "grad_norm": 56.23249435424805, |
| "learning_rate": 8.014748013882654e-07, |
| "logp/chosen": -1032.0, |
| "logp/rejected": -1800.0, |
| "loss": 0.9111328125, |
| "nll_loss": 0.57421875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.0, |
| "reward/margin": 4.59375, |
| "reward/rejected": -4.59375, |
| "step": 969 |
| }, |
| { |
| "approx. KL/chosen": 16.5, |
| "approx. KL/rejected": 128.0, |
| "epoch": 0.9938524590163934, |
| "grad_norm": 91.0499267578125, |
| "learning_rate": 8.011291628837615e-07, |
| "logp/chosen": -836.0, |
| "logp/rejected": -1104.0, |
| "loss": 1.35302734375, |
| "nll_loss": 0.52734375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.30078125, |
| "reward/margin": 1.7578125, |
| "reward/rejected": -2.046875, |
| "step": 970 |
| }, |
| { |
| "approx. KL/chosen": 78.5, |
| "approx. KL/rejected": 620.0, |
| "epoch": 0.9948770491803278, |
| "grad_norm": 57.91569900512695, |
| "learning_rate": 8.008296005639453e-07, |
| "logp/chosen": -912.0, |
| "logp/rejected": -1288.0, |
| "loss": 0.95458984375, |
| "nll_loss": 0.49609375, |
| "reward/accuracy": 0.625, |
| "reward/chosen": 0.001953125, |
| "reward/margin": 3.78125, |
| "reward/rejected": -3.78125, |
| "step": 971 |
| }, |
| { |
| "approx. KL/chosen": 42.25, |
| "approx. KL/rejected": 50.0, |
| "epoch": 0.9959016393442623, |
| "grad_norm": 50.74705123901367, |
| "learning_rate": 8.005761182640999e-07, |
| "logp/chosen": -636.0, |
| "logp/rejected": -628.0, |
| "loss": 1.0517578125, |
| "nll_loss": 0.427734375, |
| "reward/accuracy": 0.75, |
| "reward/chosen": -0.162109375, |
| "reward/margin": 0.51171875, |
| "reward/rejected": -0.67578125, |
| "step": 972 |
| }, |
| { |
| "approx. KL/chosen": 125.0, |
| "approx. KL/rejected": 150.0, |
| "epoch": 0.9969262295081968, |
| "grad_norm": 200.17288208007812, |
| "learning_rate": 8.003687192295483e-07, |
| "logp/chosen": -1208.0, |
| "logp/rejected": -1936.0, |
| "loss": 3.142578125, |
| "nll_loss": 0.5, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -1.078125, |
| "reward/margin": -0.73046875, |
| "reward/rejected": -0.3515625, |
| "step": 973 |
| }, |
| { |
| "approx. KL/chosen": 20.25, |
| "approx. KL/rejected": 123.0, |
| "epoch": 0.9979508196721312, |
| "grad_norm": 42.136314392089844, |
| "learning_rate": 8.002074061156108e-07, |
| "logp/chosen": -808.0, |
| "logp/rejected": -1120.0, |
| "loss": 1.00927734375, |
| "nll_loss": 0.671875, |
| "reward/accuracy": 0.75, |
| "reward/chosen": 0.65234375, |
| "reward/margin": 2.53125, |
| "reward/rejected": -1.875, |
| "step": 974 |
| }, |
| { |
| "approx. KL/chosen": 45.25, |
| "approx. KL/rejected": 302.0, |
| "epoch": 0.9989754098360656, |
| "grad_norm": 54.086524963378906, |
| "learning_rate": 8.000921809875723e-07, |
| "logp/chosen": -908.0, |
| "logp/rejected": -1408.0, |
| "loss": 1.19384765625, |
| "nll_loss": 0.578125, |
| "reward/accuracy": 0.875, |
| "reward/chosen": -0.75, |
| "reward/margin": 2.90625, |
| "reward/rejected": -3.65625, |
| "step": 975 |
| }, |
| { |
| "approx. KL/chosen": 50.0, |
| "approx. KL/rejected": 149.0, |
| "epoch": 1.0, |
| "grad_norm": 101.27774810791016, |
| "learning_rate": 8.00023045320655e-07, |
| "logp/chosen": -1344.0, |
| "logp/rejected": -1672.0, |
| "loss": 1.288818359375, |
| "nll_loss": 0.60546875, |
| "reward/accuracy": 0.625, |
| "reward/chosen": -0.6015625, |
| "reward/margin": 2.15625, |
| "reward/rejected": -2.75, |
| "step": 976 |
| }, |
| { |
| "epoch": 1.0, |
| "eval_approx. KL/chosen": 40.7225, |
| "eval_approx. KL/rejected": 193.91, |
| "eval_logp/chosen": -967.12, |
| "eval_logp/rejected": -1289.28, |
| "eval_loss": 1.2641308307647705, |
| "eval_nll_loss": 0.554375, |
| "eval_reward/accuracy": 0.67, |
| "eval_reward/chosen": -0.437939453125, |
| "eval_reward/margin": 2.11810546875, |
| "eval_reward/rejected": -2.556015625, |
| "eval_runtime": 47.049, |
| "eval_samples_per_second": 4.208, |
| "eval_steps_per_second": 0.531, |
| "step": 976 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 976, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 0, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 3.010507742075945e+18, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
