| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.1365374112506827, |
| "eval_steps": 500, |
| "global_step": 1000, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.0625, |
| "epoch": 0.00013653741125068268, |
| "grad_norm": 0.02207290008664131, |
| "kl": 0.0006317789470813295, |
| "learning_rate": 1.0000000000000001e-07, |
| "loss": -0.0002, |
| "num_tokens": 11046.0, |
| "reward": 0.7437499761581421, |
| "reward_std": 0.19240379333496094, |
| "rewards/reward_fn": 0.7437499761581421, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.00027307482250136535, |
| "grad_norm": 0.02206706441938877, |
| "kl": 0.0006317789470813295, |
| "learning_rate": 2.0000000000000002e-07, |
| "loss": -0.0002, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.0005360348441172391, |
| "epoch": 0.00040961223375204805, |
| "grad_norm": 0.022166570648550987, |
| "kl": 0.0006522277640215179, |
| "learning_rate": 3.0000000000000004e-07, |
| "loss": 0.0001, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0005461496450027307, |
| "grad_norm": 0.021423812955617905, |
| "kl": 0.0006513863058899005, |
| "learning_rate": 4.0000000000000003e-07, |
| "loss": -0.0002, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 0.0005387133569456637, |
| "completion_length": 122.125, |
| "epoch": 0.0006826870562534135, |
| "grad_norm": 0.02105766348540783, |
| "kl": 0.0005707805985366576, |
| "learning_rate": 5.000000000000001e-07, |
| "loss": -0.0, |
| "num_tokens": 21978.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.0004984622646588832, |
| "epoch": 0.0008192244675040961, |
| "grad_norm": 0.021098464727401733, |
| "kl": 0.0005492573363881093, |
| "learning_rate": 6.000000000000001e-07, |
| "loss": -0.0001, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.0007210461335489526, |
| "epoch": 0.0009557618787547789, |
| "grad_norm": 0.020958311855793, |
| "kl": 0.0004925800985802198, |
| "learning_rate": 7.000000000000001e-07, |
| "loss": -0.0, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.00021701389050576836, |
| "epoch": 0.0010922992900054614, |
| "grad_norm": 0.021792054176330566, |
| "kl": 0.0005863340506948589, |
| "learning_rate": 8.000000000000001e-07, |
| "loss": -0.0, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 150.21875, |
| "epoch": 0.0012288367012561442, |
| "grad_norm": 0.025502244010567665, |
| "kl": 0.0005780151559520164, |
| "learning_rate": 9.000000000000001e-07, |
| "loss": -0.0001, |
| "num_tokens": 34097.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.00046690867748111486, |
| "epoch": 0.001365374112506827, |
| "grad_norm": 0.025376036763191223, |
| "kl": 0.0005691521364497021, |
| "learning_rate": 1.0000000000000002e-06, |
| "loss": -0.0001, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0015019115237575095, |
| "grad_norm": 0.025907186791300774, |
| "kl": 0.000522814968462626, |
| "learning_rate": 1.1e-06, |
| "loss": 0.0004, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.00013646288425661623, |
| "epoch": 0.0016384489350081922, |
| "grad_norm": 0.025343747809529305, |
| "kl": 0.000557012675471924, |
| "learning_rate": 1.2000000000000002e-06, |
| "loss": 0.0002, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.0009560473263263702, |
| "completion_length": 146.15625, |
| "epoch": 0.001774986346258875, |
| "grad_norm": 0.023333657532930374, |
| "kl": 0.0005626363158626191, |
| "learning_rate": 1.3e-06, |
| "loss": 0.0, |
| "num_tokens": 45898.0, |
| "reward": 0.6312500238418579, |
| "reward_std": 0.2836301922798157, |
| "rewards/reward_fn": 0.6312500238418579, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.0008510874467901886, |
| "epoch": 0.0019115237575095577, |
| "grad_norm": 0.022622594609856606, |
| "kl": 0.0005286783398332773, |
| "learning_rate": 1.4000000000000001e-06, |
| "loss": -0.0, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.0003033980610780418, |
| "epoch": 0.0020480611687602405, |
| "grad_norm": 0.022668462246656418, |
| "kl": 0.0005744425789089291, |
| "learning_rate": 1.5e-06, |
| "loss": -0.0002, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.0005991465295664966, |
| "epoch": 0.002184598580010923, |
| "grad_norm": 0.02338634803891182, |
| "kl": 0.0005220320081207319, |
| "learning_rate": 1.6000000000000001e-06, |
| "loss": -0.0001, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.0001270325155928731, |
| "completion_length": 127.4375, |
| "epoch": 0.0023211359912616056, |
| "grad_norm": 0.023932231590151787, |
| "kl": 0.0005799778509754105, |
| "learning_rate": 1.7000000000000002e-06, |
| "loss": 0.0001, |
| "num_tokens": 57488.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.0007593779591843486, |
| "epoch": 0.0024576734025122883, |
| "grad_norm": 0.023600488901138306, |
| "kl": 0.0005501173236552859, |
| "learning_rate": 1.8000000000000001e-06, |
| "loss": -0.0001, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.0002500000118743628, |
| "epoch": 0.002594210813762971, |
| "grad_norm": 0.024099135771393776, |
| "kl": 0.0005574463702942012, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": -0.0, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.0008590422658016905, |
| "epoch": 0.002730748225013654, |
| "grad_norm": 0.024080658331513405, |
| "kl": 0.0005742711964558112, |
| "learning_rate": 2.0000000000000003e-06, |
| "loss": -0.0001, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.0015966849750839174, |
| "completion_length": 176.875, |
| "epoch": 0.0028672856362643366, |
| "grad_norm": 0.01954622007906437, |
| "kl": 0.0005687026732630329, |
| "learning_rate": 2.1000000000000002e-06, |
| "loss": 0.0001, |
| "num_tokens": 70192.0, |
| "reward": 0.4906250238418579, |
| "reward_std": 0.23124998807907104, |
| "rewards/reward_fn": 0.4906250238418579, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.0005851916066603735, |
| "epoch": 0.003003823047515019, |
| "grad_norm": 0.01967364363372326, |
| "kl": 0.0005616483749690815, |
| "learning_rate": 2.2e-06, |
| "loss": 0.0, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.0010330898803658783, |
| "epoch": 0.0031403604587657017, |
| "grad_norm": 0.019373122602701187, |
| "kl": 0.0005907249005758786, |
| "learning_rate": 2.3000000000000004e-06, |
| "loss": 0.0002, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.0009514934354228899, |
| "epoch": 0.0032768978700163844, |
| "grad_norm": 0.01972307451069355, |
| "kl": 0.0005845337018399732, |
| "learning_rate": 2.4000000000000003e-06, |
| "loss": -0.0, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.0009890664659906179, |
| "completion_length": 130.21875, |
| "epoch": 0.003413435281267067, |
| "grad_norm": 0.02998182363808155, |
| "kl": 0.0005863008900632849, |
| "learning_rate": 2.5e-06, |
| "loss": 0.0002, |
| "num_tokens": 81299.0, |
| "reward": 0.578125, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.578125, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.000487977362354286, |
| "epoch": 0.00354997269251775, |
| "grad_norm": 0.030615828931331635, |
| "kl": 0.0005413657777353365, |
| "learning_rate": 2.6e-06, |
| "loss": 0.0001, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.000487977362354286, |
| "epoch": 0.0036865101037684327, |
| "grad_norm": 0.030381930992007256, |
| "kl": 0.00050132268370362, |
| "learning_rate": 2.7000000000000004e-06, |
| "loss": 0.0001, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "epoch": 0.0038230475150191155, |
| "grad_norm": 0.03030545637011528, |
| "kl": 0.0005266097541607451, |
| "learning_rate": 2.8000000000000003e-06, |
| "loss": -0.0001, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.00014671361714135855, |
| "completion_length": 128.8125, |
| "epoch": 0.003959584926269798, |
| "grad_norm": 0.017806528136134148, |
| "kl": 0.0006746522994944826, |
| "learning_rate": 2.9e-06, |
| "loss": -0.0001, |
| "num_tokens": 92649.0, |
| "reward": 0.5437500476837158, |
| "reward_std": 0.13712069392204285, |
| "rewards/reward_fn": 0.5437500476837158, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.004096122337520481, |
| "grad_norm": 0.017719408497214317, |
| "kl": 0.0006834448263361992, |
| "learning_rate": 3e-06, |
| "loss": -0.0001, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.00031565656536258757, |
| "epoch": 0.004232659748771164, |
| "grad_norm": 0.0178105216473341, |
| "kl": 0.000749060689486214, |
| "learning_rate": 3.1000000000000004e-06, |
| "loss": -0.0001, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.00027901786961592734, |
| "epoch": 0.004369197160021846, |
| "grad_norm": 0.018075253814458847, |
| "kl": 0.0006366256338878884, |
| "learning_rate": 3.2000000000000003e-06, |
| "loss": 0.0, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.00016108246927615255, |
| "completion_length": 137.84375, |
| "epoch": 0.004505734571272528, |
| "grad_norm": 0.01944643445312977, |
| "kl": 0.0006676609509668197, |
| "learning_rate": 3.3000000000000006e-06, |
| "loss": 0.0, |
| "num_tokens": 104856.0, |
| "reward": 0.659375011920929, |
| "reward_std": 0.11874999850988388, |
| "rewards/reward_fn": 0.659375011920929, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.0006088621739763767, |
| "epoch": 0.004642271982523211, |
| "grad_norm": 0.01900452747941017, |
| "kl": 0.0006304297803580994, |
| "learning_rate": 3.4000000000000005e-06, |
| "loss": 0.0002, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.0007152466860134155, |
| "epoch": 0.004778809393773894, |
| "grad_norm": 0.019996952265501022, |
| "kl": 0.0006105903066782048, |
| "learning_rate": 3.5e-06, |
| "loss": 0.0002, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.0005187058122828603, |
| "epoch": 0.004915346805024577, |
| "grad_norm": 0.0178091861307621, |
| "kl": 0.0006019014399498701, |
| "learning_rate": 3.6000000000000003e-06, |
| "loss": -0.0001, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 148.96875, |
| "epoch": 0.005051884216275259, |
| "grad_norm": 0.00929734855890274, |
| "kl": 0.0005888168434466934, |
| "learning_rate": 3.7e-06, |
| "loss": -0.0001, |
| "num_tokens": 116419.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.005188421627525942, |
| "grad_norm": 0.009237742982804775, |
| "kl": 0.0005845859432156431, |
| "learning_rate": 3.8000000000000005e-06, |
| "loss": -0.0, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.005324959038776625, |
| "grad_norm": 0.009027848951518536, |
| "kl": 0.000577758685722074, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": -0.0001, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.005461496450027308, |
| "grad_norm": 0.00940747931599617, |
| "kl": 0.0005938013127888553, |
| "learning_rate": 4.000000000000001e-06, |
| "loss": 0.0, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.0003538882592692971, |
| "completion_length": 148.375, |
| "epoch": 0.00559803386127799, |
| "grad_norm": 0.0165647454559803, |
| "kl": 0.0005366879777284339, |
| "learning_rate": 4.1e-06, |
| "loss": 0.0001, |
| "num_tokens": 128235.0, |
| "reward": 0.8562500476837158, |
| "reward_std": 0.1711301952600479, |
| "rewards/reward_fn": 0.8562500476837158, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.0003231143855373375, |
| "epoch": 0.005734571272528673, |
| "grad_norm": 0.016383424401283264, |
| "kl": 0.0005500864490386448, |
| "learning_rate": 4.2000000000000004e-06, |
| "loss": -0.0001, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.00040967948007164523, |
| "epoch": 0.005871108683779356, |
| "grad_norm": 0.016405031085014343, |
| "kl": 0.0005530345652005053, |
| "learning_rate": 4.3e-06, |
| "loss": 0.0, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.00032404610828962177, |
| "epoch": 0.006007646095030038, |
| "grad_norm": 0.016346527263522148, |
| "kl": 0.0005656518319483439, |
| "learning_rate": 4.4e-06, |
| "loss": 0.0, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.0004112554161110893, |
| "completion_length": 137.0, |
| "epoch": 0.006144183506280721, |
| "grad_norm": 0.01740082912147045, |
| "kl": 0.000573518580495147, |
| "learning_rate": 4.5e-06, |
| "loss": -0.0, |
| "num_tokens": 140159.0, |
| "reward": 0.518750011920929, |
| "reward_std": 0.12358209490776062, |
| "rewards/reward_fn": 0.518750011920929, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.000101461038866546, |
| "epoch": 0.006280720917531403, |
| "grad_norm": 0.017511451616883278, |
| "kl": 0.0004865345692905976, |
| "learning_rate": 4.600000000000001e-06, |
| "loss": 0.0001, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.0003097943772445433, |
| "epoch": 0.006417258328782086, |
| "grad_norm": 0.017639311030507088, |
| "kl": 0.0005538275008802884, |
| "learning_rate": 4.7e-06, |
| "loss": 0.0, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.00045951134961796924, |
| "epoch": 0.006553795740032769, |
| "grad_norm": 0.017566056922078133, |
| "kl": 0.0005790123971110006, |
| "learning_rate": 4.800000000000001e-06, |
| "loss": 0.0, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.0002090300986310467, |
| "completion_length": 159.25, |
| "epoch": 0.006690333151283452, |
| "grad_norm": 0.01749393157660961, |
| "kl": 0.0005347986639208102, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": -0.0, |
| "num_tokens": 152155.0, |
| "reward": 0.4625000059604645, |
| "reward_std": 0.17499999701976776, |
| "rewards/reward_fn": 0.4625000059604645, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.00040434260154142976, |
| "epoch": 0.006826870562534134, |
| "grad_norm": 0.017458247020840645, |
| "kl": 0.0005386869147514517, |
| "learning_rate": 5e-06, |
| "loss": 0.0001, |
| "step": 50 |
| }, |
| { |
| "clip_ratio": 0.0007310814908123575, |
| "epoch": 0.006963407973784817, |
| "grad_norm": 0.017301850020885468, |
| "kl": 0.0006885109469294548, |
| "learning_rate": 4.9947368421052636e-06, |
| "loss": 0.0, |
| "step": 51 |
| }, |
| { |
| "clip_ratio": 0.00043346243182895705, |
| "epoch": 0.0070999453850355, |
| "grad_norm": 0.01740550808608532, |
| "kl": 0.0005350831738724082, |
| "learning_rate": 4.989473684210527e-06, |
| "loss": 0.0001, |
| "step": 52 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 154.03125, |
| "epoch": 0.007236482796286183, |
| "grad_norm": 0.028556713834404945, |
| "kl": 0.0005652442723658169, |
| "learning_rate": 4.98421052631579e-06, |
| "loss": 0.0002, |
| "num_tokens": 163832.0, |
| "reward": 0.3531250059604645, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.3531250059604645, |
| "step": 53 |
| }, |
| { |
| "clip_ratio": 0.0002604968976811506, |
| "epoch": 0.007373020207536865, |
| "grad_norm": 0.028979483991861343, |
| "kl": 0.0005701856089217472, |
| "learning_rate": 4.978947368421053e-06, |
| "loss": -0.0, |
| "step": 54 |
| }, |
| { |
| "clip_ratio": 0.00026939655072055757, |
| "epoch": 0.007509557618787548, |
| "grad_norm": 0.028597978875041008, |
| "kl": 0.0005870076556675485, |
| "learning_rate": 4.973684210526316e-06, |
| "loss": 0.0001, |
| "step": 55 |
| }, |
| { |
| "clip_ratio": 0.0001426940580131486, |
| "epoch": 0.007646095030038231, |
| "grad_norm": 0.02883176878094673, |
| "kl": 0.0005412003038145485, |
| "learning_rate": 4.968421052631579e-06, |
| "loss": -0.0002, |
| "step": 56 |
| }, |
| { |
| "clip_ratio": 0.000891457311809063, |
| "completion_length": 150.71875, |
| "epoch": 0.007782632441288913, |
| "grad_norm": 0.029951218515634537, |
| "kl": 0.0005701104482795927, |
| "learning_rate": 4.9631578947368426e-06, |
| "loss": 0.0001, |
| "num_tokens": 176059.0, |
| "reward": 0.578125, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.578125, |
| "step": 57 |
| }, |
| { |
| "clip_ratio": 0.0006488768121926114, |
| "epoch": 0.007919169852539596, |
| "grad_norm": 0.029265111312270164, |
| "kl": 0.000576867132622283, |
| "learning_rate": 4.957894736842106e-06, |
| "loss": -0.0001, |
| "step": 58 |
| }, |
| { |
| "clip_ratio": 0.0006633551238337532, |
| "epoch": 0.008055707263790278, |
| "grad_norm": 0.029740851372480392, |
| "kl": 0.0005770292154920753, |
| "learning_rate": 4.952631578947369e-06, |
| "loss": 0.0001, |
| "step": 59 |
| }, |
| { |
| "clip_ratio": 0.0007974752952577546, |
| "epoch": 0.008192244675040962, |
| "grad_norm": 0.029165882617235184, |
| "kl": 0.0006021304664045601, |
| "learning_rate": 4.947368421052632e-06, |
| "loss": 0.0, |
| "step": 60 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 141.15625, |
| "epoch": 0.008328782086291644, |
| "grad_norm": 0.02565835975110531, |
| "kl": 0.0005988110287944437, |
| "learning_rate": 4.942105263157895e-06, |
| "loss": -0.0002, |
| "num_tokens": 188028.0, |
| "reward": 0.859375, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.859375, |
| "step": 61 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.008465319497542327, |
| "grad_norm": 0.024776969105005264, |
| "kl": 0.0006643234455623315, |
| "learning_rate": 4.936842105263158e-06, |
| "loss": 0.0, |
| "step": 62 |
| }, |
| { |
| "clip_ratio": 0.0006443567835958675, |
| "epoch": 0.00860185690879301, |
| "grad_norm": 0.025867076590657234, |
| "kl": 0.0006416208398150047, |
| "learning_rate": 4.9315789473684215e-06, |
| "loss": -0.0002, |
| "step": 63 |
| }, |
| { |
| "clip_ratio": 0.0002910231560235843, |
| "epoch": 0.008738394320043691, |
| "grad_norm": 0.026004023849964142, |
| "kl": 0.0006436795183617505, |
| "learning_rate": 4.926315789473685e-06, |
| "loss": -0.0, |
| "step": 64 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 129.90625, |
| "epoch": 0.008874931731294375, |
| "grad_norm": 0.019275352358818054, |
| "kl": 0.0006075235532989609, |
| "learning_rate": 4.921052631578948e-06, |
| "loss": 0.0001, |
| "num_tokens": 199657.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 65 |
| }, |
| { |
| "clip_ratio": 0.00015862943837419152, |
| "epoch": 0.009011469142545057, |
| "grad_norm": 0.01945505477488041, |
| "kl": 0.000544819579772593, |
| "learning_rate": 4.915789473684211e-06, |
| "loss": 0.0, |
| "step": 66 |
| }, |
| { |
| "clip_ratio": 0.0003850787179544568, |
| "epoch": 0.00914800655379574, |
| "grad_norm": 0.019484233111143112, |
| "kl": 0.0005976865359116346, |
| "learning_rate": 4.910526315789474e-06, |
| "loss": -0.0, |
| "step": 67 |
| }, |
| { |
| "clip_ratio": 0.00015862943837419152, |
| "epoch": 0.009284543965046422, |
| "grad_norm": 0.019339406862854958, |
| "kl": 0.0005465979302243795, |
| "learning_rate": 4.905263157894737e-06, |
| "loss": 0.0001, |
| "step": 68 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 132.375, |
| "epoch": 0.009421081376297106, |
| "grad_norm": 0.02073509618639946, |
| "kl": 0.0005910295212743222, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": 0.0001, |
| "num_tokens": 210781.0, |
| "reward": 0.887499988079071, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.887499988079071, |
| "step": 69 |
| }, |
| { |
| "clip_ratio": 0.0002500000118743628, |
| "epoch": 0.009557618787547788, |
| "grad_norm": 0.02097729779779911, |
| "kl": 0.0006015199551256956, |
| "learning_rate": 4.894736842105264e-06, |
| "loss": 0.0001, |
| "step": 70 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.009694156198798471, |
| "grad_norm": 0.020941907539963722, |
| "kl": 0.000576648867536278, |
| "learning_rate": 4.889473684210527e-06, |
| "loss": 0.0, |
| "step": 71 |
| }, |
| { |
| "clip_ratio": 0.00012860081915277988, |
| "epoch": 0.009830693610049153, |
| "grad_norm": 0.02064090594649315, |
| "kl": 0.0005430334399534331, |
| "learning_rate": 4.88421052631579e-06, |
| "loss": 0.0, |
| "step": 72 |
| }, |
| { |
| "clip_ratio": 0.0012847917387261987, |
| "completion_length": 169.78125, |
| "epoch": 0.009967231021299837, |
| "grad_norm": 0.023972708731889725, |
| "kl": 0.0006444637119784602, |
| "learning_rate": 4.878947368421053e-06, |
| "loss": -0.0, |
| "num_tokens": 223878.0, |
| "reward": 0.578125, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.578125, |
| "step": 73 |
| }, |
| { |
| "clip_ratio": 0.00015624999650754035, |
| "epoch": 0.010103768432550519, |
| "grad_norm": 0.023986345157027245, |
| "kl": 0.000609115405495686, |
| "learning_rate": 4.873684210526316e-06, |
| "loss": 0.0, |
| "step": 74 |
| }, |
| { |
| "clip_ratio": 0.00039849805762059987, |
| "epoch": 0.0102403058438012, |
| "grad_norm": 0.02389119192957878, |
| "kl": 0.0005973214888399525, |
| "learning_rate": 4.8684210526315795e-06, |
| "loss": -0.0, |
| "step": 75 |
| }, |
| { |
| "clip_ratio": 0.0008600136352470145, |
| "epoch": 0.010376843255051884, |
| "grad_norm": 0.024189762771129608, |
| "kl": 0.0005895726594644657, |
| "learning_rate": 4.863157894736843e-06, |
| "loss": -0.0001, |
| "step": 76 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 144.09375, |
| "epoch": 0.010513380666302566, |
| "grad_norm": 0.017288263887166977, |
| "kl": 0.0006489278166554868, |
| "learning_rate": 4.857894736842106e-06, |
| "loss": -0.0, |
| "num_tokens": 235793.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 77 |
| }, |
| { |
| "clip_ratio": 0.000240384615608491, |
| "epoch": 0.01064991807755325, |
| "grad_norm": 0.01712276227772236, |
| "kl": 0.0005898008203075733, |
| "learning_rate": 4.852631578947369e-06, |
| "loss": 0.0, |
| "step": 78 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.010786455488803932, |
| "grad_norm": 0.01740402728319168, |
| "kl": 0.0005718218494621397, |
| "learning_rate": 4.847368421052631e-06, |
| "loss": -0.0, |
| "step": 79 |
| }, |
| { |
| "clip_ratio": 0.00012450199574232101, |
| "epoch": 0.010922992900054615, |
| "grad_norm": 0.017095359042286873, |
| "kl": 0.0006164813421491999, |
| "learning_rate": 4.842105263157895e-06, |
| "loss": -0.0001, |
| "step": 80 |
| }, |
| { |
| "clip_ratio": 0.00046910810488043353, |
| "completion_length": 131.34375, |
| "epoch": 0.011059530311305297, |
| "grad_norm": 0.02193499356508255, |
| "kl": 0.0006088345667194517, |
| "learning_rate": 4.8368421052631585e-06, |
| "loss": 0.0, |
| "num_tokens": 246760.0, |
| "reward": 0.7718750238418579, |
| "reward_std": 0.18124999105930328, |
| "rewards/reward_fn": 0.7718750238418579, |
| "step": 81 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.01119606772255598, |
| "grad_norm": 0.021939560770988464, |
| "kl": 0.0006533018759000697, |
| "learning_rate": 4.831578947368422e-06, |
| "loss": 0.0, |
| "step": 82 |
| }, |
| { |
| "clip_ratio": 0.000101461038866546, |
| "epoch": 0.011332605133806663, |
| "grad_norm": 0.02191118523478508, |
| "kl": 0.0006190983283431706, |
| "learning_rate": 4.826315789473685e-06, |
| "loss": 0.0, |
| "step": 83 |
| }, |
| { |
| "clip_ratio": 0.00047129648010013625, |
| "epoch": 0.011469142545057346, |
| "grad_norm": 0.021944301202893257, |
| "kl": 0.000652034742415708, |
| "learning_rate": 4.821052631578948e-06, |
| "loss": 0.0001, |
| "step": 84 |
| }, |
| { |
| "clip_ratio": 0.0007275540265254676, |
| "completion_length": 154.1875, |
| "epoch": 0.011605679956308028, |
| "grad_norm": 0.0288760494440794, |
| "kl": 0.0006393879930328694, |
| "learning_rate": 4.815789473684211e-06, |
| "loss": 0.0001, |
| "num_tokens": 258610.0, |
| "reward": 0.8031249642372131, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.8031249642372131, |
| "step": 85 |
| }, |
| { |
| "clip_ratio": 0.0006221036019269377, |
| "epoch": 0.011742217367558712, |
| "grad_norm": 0.028509872034192085, |
| "kl": 0.0006500086219602963, |
| "learning_rate": 4.8105263157894735e-06, |
| "loss": -0.0, |
| "step": 86 |
| }, |
| { |
| "clip_ratio": 0.001054934982676059, |
| "epoch": 0.011878754778809394, |
| "grad_norm": 0.029009725898504257, |
| "kl": 0.0006744494376107468, |
| "learning_rate": 4.8052631578947375e-06, |
| "loss": -0.0001, |
| "step": 87 |
| }, |
| { |
| "clip_ratio": 0.0009002409788081422, |
| "epoch": 0.012015292190060076, |
| "grad_norm": 0.028925592079758644, |
| "kl": 0.0006717115811625263, |
| "learning_rate": 4.800000000000001e-06, |
| "loss": 0.0001, |
| "step": 88 |
| }, |
| { |
| "clip_ratio": 0.0005800188446301036, |
| "completion_length": 136.34375, |
| "epoch": 0.01215182960131076, |
| "grad_norm": 0.02483142353594303, |
| "kl": 0.0006553460525537957, |
| "learning_rate": 4.794736842105264e-06, |
| "loss": -0.0001, |
| "num_tokens": 270849.0, |
| "reward": 0.6031249761581421, |
| "reward_std": 0.2962018847465515, |
| "rewards/reward_fn": 0.6031249761581421, |
| "step": 89 |
| }, |
| { |
| "clip_ratio": 0.0015143853743211366, |
| "epoch": 0.012288367012561441, |
| "grad_norm": 0.024743258953094482, |
| "kl": 0.0006329920825010049, |
| "learning_rate": 4.789473684210527e-06, |
| "loss": 0.0, |
| "step": 90 |
| }, |
| { |
| "clip_ratio": 0.0005078124959254637, |
| "epoch": 0.012424904423812125, |
| "grad_norm": 0.025256063789129257, |
| "kl": 0.0006229320388229098, |
| "learning_rate": 4.78421052631579e-06, |
| "loss": -0.0001, |
| "step": 91 |
| }, |
| { |
| "clip_ratio": 0.0005842391110491008, |
| "epoch": 0.012561441835062807, |
| "grad_norm": 0.025601226836442947, |
| "kl": 0.0006528175799758174, |
| "learning_rate": 4.778947368421053e-06, |
| "loss": -0.0, |
| "step": 92 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 139.3125, |
| "epoch": 0.01269797924631349, |
| "grad_norm": 0.03193929046392441, |
| "kl": 0.0006130583351477981, |
| "learning_rate": 4.773684210526316e-06, |
| "loss": -0.0001, |
| "num_tokens": 282379.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.3723076283931732, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 93 |
| }, |
| { |
| "clip_ratio": 0.00044216986862011254, |
| "epoch": 0.012834516657564172, |
| "grad_norm": 0.0314643457531929, |
| "kl": 0.0006293810620263685, |
| "learning_rate": 4.76842105263158e-06, |
| "loss": -0.0005, |
| "step": 94 |
| }, |
| { |
| "clip_ratio": 0.0005735749291488901, |
| "epoch": 0.012971054068814856, |
| "grad_norm": 0.03177262842655182, |
| "kl": 0.0005880916755813814, |
| "learning_rate": 4.763157894736842e-06, |
| "loss": -0.0003, |
| "step": 95 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.013107591480065538, |
| "grad_norm": 0.03146577253937721, |
| "kl": 0.0006502543369606428, |
| "learning_rate": 4.757894736842106e-06, |
| "loss": -0.0004, |
| "step": 96 |
| }, |
| { |
| "clip_ratio": 0.00025201612152159214, |
| "completion_length": 134.71875, |
| "epoch": 0.013244128891316221, |
| "grad_norm": 0.019158681854605675, |
| "kl": 0.0005447731527965516, |
| "learning_rate": 4.752631578947369e-06, |
| "loss": -0.0001, |
| "num_tokens": 293978.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 97 |
| }, |
| { |
| "clip_ratio": 0.00025201612152159214, |
| "epoch": 0.013380666302566903, |
| "grad_norm": 0.020707514137029648, |
| "kl": 0.0005420907405095932, |
| "learning_rate": 4.747368421052632e-06, |
| "loss": 0.0002, |
| "step": 98 |
| }, |
| { |
| "clip_ratio": 0.0004881850036326796, |
| "epoch": 0.013517203713817587, |
| "grad_norm": 0.02058817818760872, |
| "kl": 0.0005488166934810579, |
| "learning_rate": 4.7421052631578954e-06, |
| "loss": 0.0, |
| "step": 99 |
| }, |
| { |
| "clip_ratio": 0.0006924755871295929, |
| "epoch": 0.013653741125068269, |
| "grad_norm": 0.020786749199032784, |
| "kl": 0.0005361115108826198, |
| "learning_rate": 4.736842105263158e-06, |
| "loss": 0.0001, |
| "step": 100 |
| }, |
| { |
| "clip_ratio": 0.00035785317595582455, |
| "completion_length": 149.21875, |
| "epoch": 0.01379027853631895, |
| "grad_norm": 0.017742682248353958, |
| "kl": 0.0006423360018743551, |
| "learning_rate": 4.731578947368422e-06, |
| "loss": -0.0002, |
| "num_tokens": 306241.0, |
| "reward": 0.6875, |
| "reward_std": 0.17499999701976776, |
| "rewards/reward_fn": 0.6875, |
| "step": 101 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.013926815947569634, |
| "grad_norm": 0.018075380474328995, |
| "kl": 0.000610419586791977, |
| "learning_rate": 4.726315789473684e-06, |
| "loss": -0.0, |
| "step": 102 |
| }, |
| { |
| "clip_ratio": 0.0002297794126207009, |
| "epoch": 0.014063353358820316, |
| "grad_norm": 0.017695877701044083, |
| "kl": 0.0006176056695039733, |
| "learning_rate": 4.721052631578948e-06, |
| "loss": -0.0, |
| "step": 103 |
| }, |
| { |
| "clip_ratio": 0.00027076782134827226, |
| "epoch": 0.014199890770071, |
| "grad_norm": 0.01795818656682968, |
| "kl": 0.0006254155405258643, |
| "learning_rate": 4.71578947368421e-06, |
| "loss": -0.0001, |
| "step": 104 |
| }, |
| { |
| "clip_ratio": 0.0004153236513957381, |
| "completion_length": 128.25, |
| "epoch": 0.014336428181321682, |
| "grad_norm": 0.022969288751482964, |
| "kl": 0.0007000857394814375, |
| "learning_rate": 4.710526315789474e-06, |
| "loss": 0.0002, |
| "num_tokens": 317705.0, |
| "reward": 0.859375, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.859375, |
| "step": 105 |
| }, |
| { |
| "clip_ratio": 0.0004153236513957381, |
| "epoch": 0.014472965592572365, |
| "grad_norm": 0.023185281082987785, |
| "kl": 0.0006248567869988619, |
| "learning_rate": 4.705263157894738e-06, |
| "loss": -0.0, |
| "step": 106 |
| }, |
| { |
| "clip_ratio": 0.0012351995101198554, |
| "epoch": 0.014609503003823047, |
| "grad_norm": 0.021740928292274475, |
| "kl": 0.0006979771355872799, |
| "learning_rate": 4.7e-06, |
| "loss": 0.0001, |
| "step": 107 |
| }, |
| { |
| "clip_ratio": 0.0011130489874631166, |
| "epoch": 0.01474604041507373, |
| "grad_norm": 0.02287100814282894, |
| "kl": 0.000650095523269556, |
| "learning_rate": 4.694736842105264e-06, |
| "loss": 0.0001, |
| "step": 108 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 145.3125, |
| "epoch": 0.014882577826324413, |
| "grad_norm": 0.01759045571088791, |
| "kl": 0.0006179187330417335, |
| "learning_rate": 4.689473684210526e-06, |
| "loss": -0.0001, |
| "num_tokens": 329695.0, |
| "reward": 0.8031249642372131, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.8031249642372131, |
| "step": 109 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.015019115237575096, |
| "grad_norm": 0.018014615401625633, |
| "kl": 0.0006711829173582373, |
| "learning_rate": 4.68421052631579e-06, |
| "loss": 0.0001, |
| "step": 110 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.015155652648825778, |
| "grad_norm": 0.018217777833342552, |
| "kl": 0.0006618361967412056, |
| "learning_rate": 4.6789473684210525e-06, |
| "loss": 0.0002, |
| "step": 111 |
| }, |
| { |
| "clip_ratio": 0.0002281021879753098, |
| "epoch": 0.015292190060076462, |
| "grad_norm": 0.01793798618018627, |
| "kl": 0.0006607132654607994, |
| "learning_rate": 4.6736842105263166e-06, |
| "loss": 0.0001, |
| "step": 112 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 139.34375, |
| "epoch": 0.015428727471327144, |
| "grad_norm": 0.010602413676679134, |
| "kl": 0.0006683868668915238, |
| "learning_rate": 4.668421052631579e-06, |
| "loss": -0.0001, |
| "num_tokens": 341294.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 113 |
| }, |
| { |
| "clip_ratio": 0.0003324467979837209, |
| "epoch": 0.015565264882577826, |
| "grad_norm": 0.010346218943595886, |
| "kl": 0.000636374299574527, |
| "learning_rate": 4.663157894736842e-06, |
| "loss": 0.0, |
| "step": 114 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.015701802293828507, |
| "grad_norm": 0.010380822233855724, |
| "kl": 0.0005902485654587508, |
| "learning_rate": 4.657894736842106e-06, |
| "loss": -0.0, |
| "step": 115 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.015838339705079193, |
| "grad_norm": 0.010410645976662636, |
| "kl": 0.0006477099495896255, |
| "learning_rate": 4.652631578947368e-06, |
| "loss": -0.0, |
| "step": 116 |
| }, |
| { |
| "clip_ratio": 0.0007784575573168695, |
| "completion_length": 121.3125, |
| "epoch": 0.015974877116329875, |
| "grad_norm": 0.025988347828388214, |
| "kl": 0.0006193161070768838, |
| "learning_rate": 4.647368421052632e-06, |
| "loss": 0.0002, |
| "num_tokens": 352216.0, |
| "reward": 0.46562498807907104, |
| "reward_std": 0.298653781414032, |
| "rewards/reward_fn": 0.46562498807907104, |
| "step": 117 |
| }, |
| { |
| "clip_ratio": 0.0006022804300300777, |
| "epoch": 0.016111414527580557, |
| "grad_norm": 0.0258104857057333, |
| "kl": 0.0006151870020403294, |
| "learning_rate": 4.642105263157895e-06, |
| "loss": 0.0002, |
| "step": 118 |
| }, |
| { |
| "clip_ratio": 0.0004505813994910568, |
| "epoch": 0.01624795193883124, |
| "grad_norm": 0.02582407183945179, |
| "kl": 0.0006998937201387889, |
| "learning_rate": 4.636842105263159e-06, |
| "loss": 0.0, |
| "step": 119 |
| }, |
| { |
| "clip_ratio": 0.0004073779273312539, |
| "epoch": 0.016384489350081924, |
| "grad_norm": 0.02589990384876728, |
| "kl": 0.0005731130522690364, |
| "learning_rate": 4.631578947368421e-06, |
| "loss": 0.0002, |
| "step": 120 |
| }, |
| { |
| "clip_ratio": 0.00043370435014367104, |
| "completion_length": 146.9375, |
| "epoch": 0.016521026761332606, |
| "grad_norm": 0.020061973482370377, |
| "kl": 0.0006172579032863723, |
| "learning_rate": 4.626315789473684e-06, |
| "loss": 0.0002, |
| "num_tokens": 363906.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 121 |
| }, |
| { |
| "clip_ratio": 0.0007088417623890564, |
| "epoch": 0.016657564172583288, |
| "grad_norm": 0.019345788285136223, |
| "kl": 0.0006093144372698589, |
| "learning_rate": 4.621052631578948e-06, |
| "loss": 0.0002, |
| "step": 122 |
| }, |
| { |
| "clip_ratio": 0.0002741228090599179, |
| "epoch": 0.01679410158383397, |
| "grad_norm": 0.01971699669957161, |
| "kl": 0.000635543535281613, |
| "learning_rate": 4.6157894736842105e-06, |
| "loss": 0.0002, |
| "step": 123 |
| }, |
| { |
| "clip_ratio": 0.0009507163631496951, |
| "epoch": 0.016930638995084655, |
| "grad_norm": 0.019793082028627396, |
| "kl": 0.0005919806335441535, |
| "learning_rate": 4.6105263157894745e-06, |
| "loss": 0.0001, |
| "step": 124 |
| }, |
| { |
| "clip_ratio": 0.0004717841511592269, |
| "completion_length": 128.40625, |
| "epoch": 0.017067176406335337, |
| "grad_norm": 0.027957087382674217, |
| "kl": 0.0006280076445364102, |
| "learning_rate": 4.605263157894737e-06, |
| "loss": -0.0002, |
| "num_tokens": 375419.0, |
| "reward": 0.46562501788139343, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.46562501788139343, |
| "step": 125 |
| }, |
| { |
| "clip_ratio": 0.00044455942406784743, |
| "epoch": 0.01720371381758602, |
| "grad_norm": 0.026606649160385132, |
| "kl": 0.0005630467830997077, |
| "learning_rate": 4.600000000000001e-06, |
| "loss": -0.0001, |
| "step": 126 |
| }, |
| { |
| "clip_ratio": 0.0004891337011940777, |
| "epoch": 0.0173402512288367, |
| "grad_norm": 0.027467843145132065, |
| "kl": 0.0006166973080325988, |
| "learning_rate": 4.594736842105263e-06, |
| "loss": -0.0002, |
| "step": 127 |
| }, |
| { |
| "clip_ratio": 0.0004891337011940777, |
| "epoch": 0.017476788640087382, |
| "grad_norm": 0.027233824133872986, |
| "kl": 0.0006092134076425282, |
| "learning_rate": 4.589473684210526e-06, |
| "loss": -0.0002, |
| "step": 128 |
| }, |
| { |
| "clip_ratio": 0.00046258504153229296, |
| "completion_length": 129.625, |
| "epoch": 0.017613326051338068, |
| "grad_norm": 0.018424691632390022, |
| "kl": 0.0006046048201824306, |
| "learning_rate": 4.5842105263157895e-06, |
| "loss": -0.0001, |
| "num_tokens": 386775.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 129 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.01774986346258875, |
| "grad_norm": 0.0189372468739748, |
| "kl": 0.0006345385913846258, |
| "learning_rate": 4.578947368421053e-06, |
| "loss": -0.0001, |
| "step": 130 |
| }, |
| { |
| "clip_ratio": 0.00021258502965793014, |
| "epoch": 0.01788640087383943, |
| "grad_norm": 0.018727825954556465, |
| "kl": 0.0006059091974748299, |
| "learning_rate": 4.573684210526317e-06, |
| "loss": -0.0001, |
| "step": 131 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.018022938285090113, |
| "grad_norm": 0.01906794309616089, |
| "kl": 0.0005894042797081056, |
| "learning_rate": 4.568421052631579e-06, |
| "loss": -0.0002, |
| "step": 132 |
| }, |
| { |
| "clip_ratio": 0.0006876413826830685, |
| "completion_length": 124.3125, |
| "epoch": 0.0181594756963408, |
| "grad_norm": 0.023546243086457253, |
| "kl": 0.0004997413302589848, |
| "learning_rate": 4.563157894736843e-06, |
| "loss": -0.0001, |
| "num_tokens": 397609.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 133 |
| }, |
| { |
| "clip_ratio": 0.0005090699560241774, |
| "epoch": 0.01829601310759148, |
| "grad_norm": 0.02335953712463379, |
| "kl": 0.0005571195488300873, |
| "learning_rate": 4.557894736842105e-06, |
| "loss": -0.0001, |
| "step": 134 |
| }, |
| { |
| "clip_ratio": 0.0006317299848888069, |
| "epoch": 0.018432550518842163, |
| "grad_norm": 0.02356523647904396, |
| "kl": 0.0005225548211456044, |
| "learning_rate": 4.552631578947369e-06, |
| "loss": -0.0001, |
| "step": 135 |
| }, |
| { |
| "clip_ratio": 0.0005343041266314685, |
| "epoch": 0.018569087930092845, |
| "grad_norm": 0.023870345205068588, |
| "kl": 0.0005025722261962073, |
| "learning_rate": 4.547368421052632e-06, |
| "loss": -0.0001, |
| "step": 136 |
| }, |
| { |
| "clip_ratio": 0.0001502403902122751, |
| "completion_length": 153.5, |
| "epoch": 0.01870562534134353, |
| "grad_norm": 0.012812867760658264, |
| "kl": 0.0006005578643453191, |
| "learning_rate": 4.542105263157895e-06, |
| "loss": 0.0001, |
| "num_tokens": 409909.0, |
| "reward": 0.9437500238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.9437500238418579, |
| "step": 137 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.018842162752594212, |
| "grad_norm": 0.012940590269863605, |
| "kl": 0.0005563261474890169, |
| "learning_rate": 4.536842105263158e-06, |
| "loss": 0.0001, |
| "step": 138 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.018978700163844894, |
| "grad_norm": 0.012597217224538326, |
| "kl": 0.0006045326954335906, |
| "learning_rate": 4.531578947368421e-06, |
| "loss": 0.0, |
| "step": 139 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.019115237575095576, |
| "grad_norm": 0.012925639748573303, |
| "kl": 0.0005933965840085875, |
| "learning_rate": 4.526315789473685e-06, |
| "loss": -0.0001, |
| "step": 140 |
| }, |
| { |
| "clip_ratio": 0.0009048875363077968, |
| "completion_length": 150.78125, |
| "epoch": 0.019251774986346257, |
| "grad_norm": 0.02253352664411068, |
| "kl": 0.0005975085450700135, |
| "learning_rate": 4.5210526315789475e-06, |
| "loss": -0.0001, |
| "num_tokens": 422358.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.578125, |
| "step": 141 |
| }, |
| { |
| "clip_ratio": 0.00021114865376148373, |
| "epoch": 0.019388312397596943, |
| "grad_norm": 0.021753521636128426, |
| "kl": 0.0006178569246912957, |
| "learning_rate": 4.5157894736842115e-06, |
| "loss": -0.0004, |
| "step": 142 |
| }, |
| { |
| "clip_ratio": 0.00021114865376148373, |
| "epoch": 0.019524849808847625, |
| "grad_norm": 0.022238247096538544, |
| "kl": 0.0005942240950389532, |
| "learning_rate": 4.510526315789474e-06, |
| "loss": -0.0002, |
| "step": 143 |
| }, |
| { |
| "clip_ratio": 0.00021114865376148373, |
| "epoch": 0.019661387220098307, |
| "grad_norm": 0.022495048120617867, |
| "kl": 0.000565216106679145, |
| "learning_rate": 4.505263157894737e-06, |
| "loss": -0.0001, |
| "step": 144 |
| }, |
| { |
| "clip_ratio": 0.0006272522732615471, |
| "completion_length": 121.375, |
| "epoch": 0.01979792463134899, |
| "grad_norm": 0.025133801624178886, |
| "kl": 0.0006039909007995448, |
| "learning_rate": 4.5e-06, |
| "loss": 0.0, |
| "num_tokens": 432838.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 145 |
| }, |
| { |
| "clip_ratio": 0.0011298577737761661, |
| "epoch": 0.019934462042599674, |
| "grad_norm": 0.025172198191285133, |
| "kl": 0.0005761020265708794, |
| "learning_rate": 4.494736842105263e-06, |
| "loss": -0.0001, |
| "step": 146 |
| }, |
| { |
| "clip_ratio": 0.0006077199795981869, |
| "epoch": 0.020070999453850356, |
| "grad_norm": 0.02526123821735382, |
| "kl": 0.0006754654223186662, |
| "learning_rate": 4.489473684210527e-06, |
| "loss": -0.0, |
| "step": 147 |
| }, |
| { |
| "clip_ratio": 0.0004832089616684243, |
| "epoch": 0.020207536865101038, |
| "grad_norm": 0.02527388371527195, |
| "kl": 0.0007189459374785656, |
| "learning_rate": 4.48421052631579e-06, |
| "loss": 0.0001, |
| "step": 148 |
| }, |
| { |
| "clip_ratio": 0.00026939655072055757, |
| "completion_length": 153.34375, |
| "epoch": 0.02034407427635172, |
| "grad_norm": 0.02411748096346855, |
| "kl": 0.0006902036120663979, |
| "learning_rate": 4.478947368421054e-06, |
| "loss": -0.0001, |
| "num_tokens": 444481.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 149 |
| }, |
| { |
| "clip_ratio": 0.0006806772944401018, |
| "epoch": 0.0204806116876024, |
| "grad_norm": 0.024839146062731743, |
| "kl": 0.0006397511897375807, |
| "learning_rate": 4.473684210526316e-06, |
| "loss": -0.0004, |
| "step": 150 |
| }, |
| { |
| "clip_ratio": 0.0005650091770803556, |
| "epoch": 0.020617149098853087, |
| "grad_norm": 0.02479824796319008, |
| "kl": 0.0006595764225494349, |
| "learning_rate": 4.468421052631579e-06, |
| "loss": -0.0001, |
| "step": 151 |
| }, |
| { |
| "clip_ratio": 0.0005590896907960996, |
| "epoch": 0.02075368651010377, |
| "grad_norm": 0.024913184344768524, |
| "kl": 0.0006492447819255176, |
| "learning_rate": 4.463157894736842e-06, |
| "loss": -0.0003, |
| "step": 152 |
| }, |
| { |
| "clip_ratio": 0.0008531439525540918, |
| "completion_length": 133.96875, |
| "epoch": 0.02089022392135445, |
| "grad_norm": 0.016449004411697388, |
| "kl": 0.000623302158601291, |
| "learning_rate": 4.4578947368421054e-06, |
| "loss": -0.0001, |
| "num_tokens": 455948.0, |
| "reward": 0.4375, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.4375, |
| "step": 153 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.021026761332605132, |
| "grad_norm": 0.016440996900200844, |
| "kl": 0.0006272347632148012, |
| "learning_rate": 4.452631578947369e-06, |
| "loss": -0.0, |
| "step": 154 |
| }, |
| { |
| "clip_ratio": 0.00013185653369873762, |
| "epoch": 0.021163298743855818, |
| "grad_norm": 0.016483375802636147, |
| "kl": 0.0005931556556788564, |
| "learning_rate": 4.447368421052632e-06, |
| "loss": -0.0002, |
| "step": 155 |
| }, |
| { |
| "clip_ratio": 0.00013185653369873762, |
| "epoch": 0.0212998361551065, |
| "grad_norm": 0.016074543818831444, |
| "kl": 0.0005772067979705753, |
| "learning_rate": 4.442105263157896e-06, |
| "loss": -0.0002, |
| "step": 156 |
| }, |
| { |
| "clip_ratio": 0.001116631436161697, |
| "completion_length": 131.5625, |
| "epoch": 0.02143637356635718, |
| "grad_norm": 0.026868479326367378, |
| "kl": 0.000641528177766304, |
| "learning_rate": 4.436842105263158e-06, |
| "loss": 0.0002, |
| "num_tokens": 466882.0, |
| "reward": 0.6031249761581421, |
| "reward_std": 0.24478399753570557, |
| "rewards/reward_fn": 0.6031249761581421, |
| "step": 157 |
| }, |
| { |
| "clip_ratio": 0.0008329408010467887, |
| "epoch": 0.021572910977607863, |
| "grad_norm": 0.02681853622198105, |
| "kl": 0.0005750876798629179, |
| "learning_rate": 4.431578947368421e-06, |
| "loss": 0.0, |
| "step": 158 |
| }, |
| { |
| "clip_ratio": 0.0020322165510151535, |
| "epoch": 0.02170944838885855, |
| "grad_norm": 0.026626134291291237, |
| "kl": 0.000633980415386759, |
| "learning_rate": 4.426315789473684e-06, |
| "loss": 0.0002, |
| "step": 159 |
| }, |
| { |
| "clip_ratio": 0.0011721871269401163, |
| "epoch": 0.02184598580010923, |
| "grad_norm": 0.026342855766415596, |
| "kl": 0.000581728978886531, |
| "learning_rate": 4.4210526315789476e-06, |
| "loss": -0.0002, |
| "step": 160 |
| }, |
| { |
| "clip_ratio": 0.00034722223062999547, |
| "completion_length": 142.96875, |
| "epoch": 0.021982523211359913, |
| "grad_norm": 0.03223702311515808, |
| "kl": 0.000677699447805935, |
| "learning_rate": 4.415789473684211e-06, |
| "loss": -0.0001, |
| "num_tokens": 478801.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 161 |
| }, |
| { |
| "clip_ratio": 0.00045375632180366665, |
| "epoch": 0.022119060622610594, |
| "grad_norm": 0.03263309225440025, |
| "kl": 0.0006566386318809236, |
| "learning_rate": 4.410526315789474e-06, |
| "loss": -0.0, |
| "step": 162 |
| }, |
| { |
| "clip_ratio": 0.0004954312171321362, |
| "epoch": 0.022255598033861276, |
| "grad_norm": 0.03234660625457764, |
| "kl": 0.0006995576613917365, |
| "learning_rate": 4.405263157894737e-06, |
| "loss": -0.0001, |
| "step": 163 |
| }, |
| { |
| "clip_ratio": 0.0003730945463757962, |
| "epoch": 0.02239213544511196, |
| "grad_norm": 0.03226694092154503, |
| "kl": 0.000669665740133496, |
| "learning_rate": 4.4e-06, |
| "loss": 0.0001, |
| "step": 164 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 153.90625, |
| "epoch": 0.022528672856362644, |
| "grad_norm": 0.02280597761273384, |
| "kl": 0.000539449552888982, |
| "learning_rate": 4.394736842105263e-06, |
| "loss": -0.0001, |
| "num_tokens": 491206.0, |
| "reward": 0.7156250476837158, |
| "reward_std": 0.23124998807907104, |
| "rewards/reward_fn": 0.7156250476837158, |
| "step": 165 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.022665210267613325, |
| "grad_norm": 0.022639747709035873, |
| "kl": 0.0005247992294243886, |
| "learning_rate": 4.3894736842105266e-06, |
| "loss": -0.0001, |
| "step": 166 |
| }, |
| { |
| "clip_ratio": 0.0003004807804245502, |
| "epoch": 0.022801747678864007, |
| "grad_norm": 0.023010019212961197, |
| "kl": 0.0005579576090894989, |
| "learning_rate": 4.38421052631579e-06, |
| "loss": 0.0, |
| "step": 167 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.022938285090114693, |
| "grad_norm": 0.023213736712932587, |
| "kl": 0.0005310979404384852, |
| "learning_rate": 4.378947368421053e-06, |
| "loss": 0.0001, |
| "step": 168 |
| }, |
| { |
| "clip_ratio": 0.000481564158690162, |
| "completion_length": 162.25, |
| "epoch": 0.023074822501365375, |
| "grad_norm": 0.026983601972460747, |
| "kl": 0.0005834002631672774, |
| "learning_rate": 4.373684210526316e-06, |
| "loss": 0.0, |
| "num_tokens": 503338.0, |
| "reward": 0.71875, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.71875, |
| "step": 169 |
| }, |
| { |
| "clip_ratio": 0.00032799315522424877, |
| "epoch": 0.023211359912616056, |
| "grad_norm": 0.02684599533677101, |
| "kl": 0.0005691815322279581, |
| "learning_rate": 4.368421052631579e-06, |
| "loss": 0.0, |
| "step": 170 |
| }, |
| { |
| "clip_ratio": 0.0010038797772722319, |
| "epoch": 0.02334789732386674, |
| "grad_norm": 0.027076352387666702, |
| "kl": 0.0005902027933188947, |
| "learning_rate": 4.363157894736842e-06, |
| "loss": 0.0002, |
| "step": 171 |
| }, |
| { |
| "clip_ratio": 0.0005283136706566438, |
| "epoch": 0.023484434735117424, |
| "grad_norm": 0.02659190259873867, |
| "kl": 0.0005958642700534256, |
| "learning_rate": 4.3578947368421055e-06, |
| "loss": 0.0001, |
| "step": 172 |
| }, |
| { |
| "clip_ratio": 0.000557463092263788, |
| "completion_length": 137.28125, |
| "epoch": 0.023620972146368106, |
| "grad_norm": 0.019615458324551582, |
| "kl": 0.0006092825424275361, |
| "learning_rate": 4.352631578947369e-06, |
| "loss": 0.0002, |
| "num_tokens": 514483.0, |
| "reward": 0.9156249761581421, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.9156249761581421, |
| "step": 173 |
| }, |
| { |
| "clip_ratio": 0.0007839123718440533, |
| "epoch": 0.023757509557618788, |
| "grad_norm": 0.019610155373811722, |
| "kl": 0.0006007257366036356, |
| "learning_rate": 4.347368421052632e-06, |
| "loss": 0.0002, |
| "step": 174 |
| }, |
| { |
| "clip_ratio": 0.0003033980610780418, |
| "epoch": 0.02389404696886947, |
| "grad_norm": 0.01908615045249462, |
| "kl": 0.0006212977809809672, |
| "learning_rate": 4.342105263157895e-06, |
| "loss": 0.0002, |
| "step": 175 |
| }, |
| { |
| "clip_ratio": 0.001106989337131381, |
| "epoch": 0.02403058438012015, |
| "grad_norm": 0.01965913362801075, |
| "kl": 0.0006572387419510051, |
| "learning_rate": 4.336842105263158e-06, |
| "loss": 0.0001, |
| "step": 176 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 148.5, |
| "epoch": 0.024167121791370837, |
| "grad_norm": 0.02616649493575096, |
| "kl": 0.0006305564247668372, |
| "learning_rate": 4.331578947368421e-06, |
| "loss": -0.0001, |
| "num_tokens": 526339.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 177 |
| }, |
| { |
| "clip_ratio": 0.0005066010635346174, |
| "epoch": 0.02430365920262152, |
| "grad_norm": 0.02656235173344612, |
| "kl": 0.000589995510836161, |
| "learning_rate": 4.3263157894736845e-06, |
| "loss": 0.0, |
| "step": 178 |
| }, |
| { |
| "clip_ratio": 0.0006540067261084914, |
| "epoch": 0.0244401966138722, |
| "grad_norm": 0.025882191956043243, |
| "kl": 0.0006225678325790795, |
| "learning_rate": 4.321052631578948e-06, |
| "loss": -0.0001, |
| "step": 179 |
| }, |
| { |
| "clip_ratio": 0.000147405662573874, |
| "epoch": 0.024576734025122882, |
| "grad_norm": 0.026141585782170296, |
| "kl": 0.0007103291836756398, |
| "learning_rate": 4.315789473684211e-06, |
| "loss": -0.0002, |
| "step": 180 |
| }, |
| { |
| "clip_ratio": 0.0008024979470064864, |
| "completion_length": 136.84375, |
| "epoch": 0.024713271436373568, |
| "grad_norm": 0.02634594403207302, |
| "kl": 0.0006556506277775043, |
| "learning_rate": 4.310526315789474e-06, |
| "loss": -0.0001, |
| "num_tokens": 538294.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 181 |
| }, |
| { |
| "clip_ratio": 0.0003042656317120418, |
| "epoch": 0.02484980884762425, |
| "grad_norm": 0.02641279436647892, |
| "kl": 0.0005787361596958362, |
| "learning_rate": 4.305263157894737e-06, |
| "loss": -0.0001, |
| "step": 182 |
| }, |
| { |
| "clip_ratio": 0.0002822835522238165, |
| "epoch": 0.02498634625887493, |
| "grad_norm": 0.026922186836600304, |
| "kl": 0.0006166264679450251, |
| "learning_rate": 4.3e-06, |
| "loss": -0.0002, |
| "step": 183 |
| }, |
| { |
| "clip_ratio": 0.00028801843291148543, |
| "epoch": 0.025122883670125613, |
| "grad_norm": 0.02697991020977497, |
| "kl": 0.0006883774053676461, |
| "learning_rate": 4.2947368421052635e-06, |
| "loss": -0.0002, |
| "step": 184 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 149.96875, |
| "epoch": 0.0252594210813763, |
| "grad_norm": 0.014164031483232975, |
| "kl": 0.0006083018552089925, |
| "learning_rate": 4.289473684210527e-06, |
| "loss": 0.0, |
| "num_tokens": 550441.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 185 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.02539595849262698, |
| "grad_norm": 0.014139293693006039, |
| "kl": 0.0006685331882181345, |
| "learning_rate": 4.28421052631579e-06, |
| "loss": 0.0, |
| "step": 186 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.025532495903877662, |
| "grad_norm": 0.014367860741913319, |
| "kl": 0.0006282098470364872, |
| "learning_rate": 4.278947368421053e-06, |
| "loss": 0.0001, |
| "step": 187 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.025669033315128344, |
| "grad_norm": 0.014389986172318459, |
| "kl": 0.0006063026085030288, |
| "learning_rate": 4.273684210526316e-06, |
| "loss": 0.0, |
| "step": 188 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 118.75, |
| "epoch": 0.025805570726379026, |
| "grad_norm": 0.016773829236626625, |
| "kl": 0.0006331677345770004, |
| "learning_rate": 4.268421052631579e-06, |
| "loss": -0.0001, |
| "num_tokens": 560797.0, |
| "reward": 0.831250011920929, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.831250011920929, |
| "step": 189 |
| }, |
| { |
| "clip_ratio": 0.00026483050896786153, |
| "epoch": 0.02594210813762971, |
| "grad_norm": 0.016798226162791252, |
| "kl": 0.000630740689302911, |
| "learning_rate": 4.2631578947368425e-06, |
| "loss": 0.0001, |
| "step": 190 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.026078645548880394, |
| "grad_norm": 0.01683247834444046, |
| "kl": 0.0007194746331151691, |
| "learning_rate": 4.257894736842106e-06, |
| "loss": 0.0, |
| "step": 191 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.026215182960131075, |
| "grad_norm": 0.017007527872920036, |
| "kl": 0.000644579552954383, |
| "learning_rate": 4.252631578947369e-06, |
| "loss": 0.0001, |
| "step": 192 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 139.1875, |
| "epoch": 0.026351720371381757, |
| "grad_norm": 0.02215302176773548, |
| "kl": 0.000574918890833942, |
| "learning_rate": 4.247368421052632e-06, |
| "loss": -0.0002, |
| "num_tokens": 571887.0, |
| "reward": 0.71875, |
| "reward_std": 0.1948557198047638, |
| "rewards/reward_fn": 0.71875, |
| "step": 193 |
| }, |
| { |
| "clip_ratio": 0.00019654087373055518, |
| "epoch": 0.026488257782632443, |
| "grad_norm": 0.021721050143241882, |
| "kl": 0.0006944113565623411, |
| "learning_rate": 4.242105263157895e-06, |
| "loss": -0.0002, |
| "step": 194 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.026624795193883125, |
| "grad_norm": 0.02209274098277092, |
| "kl": 0.0006284140536081395, |
| "learning_rate": 4.236842105263158e-06, |
| "loss": -0.0002, |
| "step": 195 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.026761332605133806, |
| "grad_norm": 0.022048382088541985, |
| "kl": 0.000593657642184553, |
| "learning_rate": 4.2315789473684215e-06, |
| "loss": -0.0002, |
| "step": 196 |
| }, |
| { |
| "clip_ratio": 0.0005545529711525887, |
| "completion_length": 124.375, |
| "epoch": 0.02689787001638449, |
| "grad_norm": 0.03627476841211319, |
| "kl": 0.0006863787939437316, |
| "learning_rate": 4.226315789473685e-06, |
| "loss": 0.0003, |
| "num_tokens": 582723.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 197 |
| }, |
| { |
| "clip_ratio": 0.0011091082269558683, |
| "epoch": 0.027034407427635174, |
| "grad_norm": 0.03692282736301422, |
| "kl": 0.0007320116324081027, |
| "learning_rate": 4.221052631578948e-06, |
| "loss": 0.0001, |
| "step": 198 |
| }, |
| { |
| "clip_ratio": 0.0007693048100918531, |
| "epoch": 0.027170944838885856, |
| "grad_norm": 0.03559679165482521, |
| "kl": 0.0007174053198468755, |
| "learning_rate": 4.215789473684211e-06, |
| "loss": 0.0003, |
| "step": 199 |
| }, |
| { |
| "clip_ratio": 0.0009161705384030938, |
| "epoch": 0.027307482250136537, |
| "grad_norm": 0.036062467843294144, |
| "kl": 0.0007688817172493145, |
| "learning_rate": 4.210526315789474e-06, |
| "loss": 0.0002, |
| "step": 200 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 132.4375, |
| "epoch": 0.02744401966138722, |
| "grad_norm": 0.01810489036142826, |
| "kl": 0.0006160379116408876, |
| "learning_rate": 4.205263157894737e-06, |
| "loss": -0.0001, |
| "num_tokens": 594265.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 201 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0275805570726379, |
| "grad_norm": 0.017982803285121918, |
| "kl": 0.0005821473964715551, |
| "learning_rate": 4.2000000000000004e-06, |
| "loss": -0.0001, |
| "step": 202 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.027717094483888587, |
| "grad_norm": 0.018066201359033585, |
| "kl": 0.0005501055475178873, |
| "learning_rate": 4.194736842105264e-06, |
| "loss": -0.0001, |
| "step": 203 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.02785363189513927, |
| "grad_norm": 0.018188124522566795, |
| "kl": 0.0005876172319858597, |
| "learning_rate": 4.189473684210527e-06, |
| "loss": -0.0001, |
| "step": 204 |
| }, |
| { |
| "clip_ratio": 0.0003033980610780418, |
| "completion_length": 129.4375, |
| "epoch": 0.02799016930638995, |
| "grad_norm": 0.01925286278128624, |
| "kl": 0.0006736774898854492, |
| "learning_rate": 4.18421052631579e-06, |
| "loss": 0.0002, |
| "num_tokens": 605391.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 205 |
| }, |
| { |
| "clip_ratio": 0.0003033980610780418, |
| "epoch": 0.028126706717640632, |
| "grad_norm": 0.019308850169181824, |
| "kl": 0.000748576178921212, |
| "learning_rate": 4.178947368421053e-06, |
| "loss": 0.0002, |
| "step": 206 |
| }, |
| { |
| "clip_ratio": 0.0004951158480253071, |
| "epoch": 0.028263244128891318, |
| "grad_norm": 0.01930277608335018, |
| "kl": 0.0007205891324701952, |
| "learning_rate": 4.173684210526316e-06, |
| "loss": 0.0002, |
| "step": 207 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.028399781540142, |
| "grad_norm": 0.018969830125570297, |
| "kl": 0.0007646826934433193, |
| "learning_rate": 4.1684210526315794e-06, |
| "loss": 0.0, |
| "step": 208 |
| }, |
| { |
| "clip_ratio": 0.0002367424312978983, |
| "completion_length": 142.59375, |
| "epoch": 0.02853631895139268, |
| "grad_norm": 0.018274182453751564, |
| "kl": 0.0006968205607336131, |
| "learning_rate": 4.163157894736843e-06, |
| "loss": 0.0002, |
| "num_tokens": 617790.0, |
| "reward": 0.578125, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.578125, |
| "step": 209 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.028672856362643363, |
| "grad_norm": 0.018192008137702942, |
| "kl": 0.0006250791311686044, |
| "learning_rate": 4.157894736842106e-06, |
| "loss": 0.0001, |
| "step": 210 |
| }, |
| { |
| "clip_ratio": 0.0002604166802484542, |
| "epoch": 0.02880939377389405, |
| "grad_norm": 0.017575176432728767, |
| "kl": 0.0006424866801353346, |
| "learning_rate": 4.152631578947369e-06, |
| "loss": 0.0001, |
| "step": 211 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.02894593118514473, |
| "grad_norm": 0.01796414516866207, |
| "kl": 0.0006217293284862535, |
| "learning_rate": 4.147368421052632e-06, |
| "loss": 0.0, |
| "step": 212 |
| }, |
| { |
| "clip_ratio": 0.0001653439103392884, |
| "completion_length": 144.0625, |
| "epoch": 0.029082468596395412, |
| "grad_norm": 0.01483822800219059, |
| "kl": 0.0006835903727733239, |
| "learning_rate": 4.142105263157895e-06, |
| "loss": -0.0001, |
| "num_tokens": 629920.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 213 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.029219006007646094, |
| "grad_norm": 0.014619370922446251, |
| "kl": 0.0006971015118324431, |
| "learning_rate": 4.136842105263158e-06, |
| "loss": -0.0001, |
| "step": 214 |
| }, |
| { |
| "clip_ratio": 0.0001653439103392884, |
| "epoch": 0.029355543418896776, |
| "grad_norm": 0.014896603301167488, |
| "kl": 0.0007456434082087071, |
| "learning_rate": 4.1315789473684216e-06, |
| "loss": 0.0, |
| "step": 215 |
| }, |
| { |
| "clip_ratio": 0.0001653439103392884, |
| "epoch": 0.02949208083014746, |
| "grad_norm": 0.015029766596853733, |
| "kl": 0.0007792614933350706, |
| "learning_rate": 4.126315789473685e-06, |
| "loss": -0.0, |
| "step": 216 |
| }, |
| { |
| "clip_ratio": 0.0007776991260470822, |
| "completion_length": 149.4375, |
| "epoch": 0.029628618241398143, |
| "grad_norm": 0.024460721760988235, |
| "kl": 0.0006598385712095478, |
| "learning_rate": 4.121052631578948e-06, |
| "loss": -0.0, |
| "num_tokens": 642062.0, |
| "reward": 0.6031250357627869, |
| "reward_std": 0.23124998807907104, |
| "rewards/reward_fn": 0.6031250357627869, |
| "step": 217 |
| }, |
| { |
| "clip_ratio": 0.0002281021879753098, |
| "epoch": 0.029765155652648825, |
| "grad_norm": 0.024433676153421402, |
| "kl": 0.0006593063117179554, |
| "learning_rate": 4.115789473684211e-06, |
| "loss": -0.0, |
| "step": 218 |
| }, |
| { |
| "clip_ratio": 0.0013088609266560525, |
| "epoch": 0.029901693063899507, |
| "grad_norm": 0.024088183417916298, |
| "kl": 0.0007106323237167089, |
| "learning_rate": 4.110526315789474e-06, |
| "loss": -0.0001, |
| "step": 219 |
| }, |
| { |
| "clip_ratio": 0.0007659851835342124, |
| "epoch": 0.030038230475150193, |
| "grad_norm": 0.02432289533317089, |
| "kl": 0.0006990563970248331, |
| "learning_rate": 4.105263157894737e-06, |
| "loss": -0.0001, |
| "step": 220 |
| }, |
| { |
| "clip_ratio": 0.0002741228090599179, |
| "completion_length": 138.3125, |
| "epoch": 0.030174767886400874, |
| "grad_norm": 0.018672458827495575, |
| "kl": 0.0006957703599255183, |
| "learning_rate": 4.1e-06, |
| "loss": 0.0, |
| "num_tokens": 653788.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 221 |
| }, |
| { |
| "clip_ratio": 0.000294811325147748, |
| "epoch": 0.030311305297651556, |
| "grad_norm": 0.018861589953303337, |
| "kl": 0.0006577782787644537, |
| "learning_rate": 4.094736842105264e-06, |
| "loss": -0.0, |
| "step": 222 |
| }, |
| { |
| "clip_ratio": 0.0005017649673391134, |
| "epoch": 0.030447842708902238, |
| "grad_norm": 0.019544165581464767, |
| "kl": 0.0006526250754177454, |
| "learning_rate": 4.089473684210527e-06, |
| "loss": -0.0001, |
| "step": 223 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.030584380120152924, |
| "grad_norm": 0.018741516396403313, |
| "kl": 0.0006388758838511421, |
| "learning_rate": 4.08421052631579e-06, |
| "loss": -0.0, |
| "step": 224 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 143.75, |
| "epoch": 0.030720917531403606, |
| "grad_norm": 0.01722196862101555, |
| "kl": 0.0007343545703406562, |
| "learning_rate": 4.078947368421053e-06, |
| "loss": 0.0, |
| "num_tokens": 665160.0, |
| "reward": 0.7750000357627869, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7750000357627869, |
| "step": 225 |
| }, |
| { |
| "clip_ratio": 0.0002741228090599179, |
| "epoch": 0.030857454942654287, |
| "grad_norm": 0.017116790637373924, |
| "kl": 0.0007918194187368499, |
| "learning_rate": 4.073684210526316e-06, |
| "loss": -0.0, |
| "step": 226 |
| }, |
| { |
| "clip_ratio": 0.0005372232117224485, |
| "epoch": 0.03099399235390497, |
| "grad_norm": 0.017037423327565193, |
| "kl": 0.0007311694735108176, |
| "learning_rate": 4.0684210526315795e-06, |
| "loss": -0.0, |
| "step": 227 |
| }, |
| { |
| "clip_ratio": 0.0006525368662551045, |
| "epoch": 0.03113052976515565, |
| "grad_norm": 0.01683720573782921, |
| "kl": 0.0007448554715665523, |
| "learning_rate": 4.063157894736842e-06, |
| "loss": -0.0, |
| "step": 228 |
| }, |
| { |
| "clip_ratio": 0.000498970810440369, |
| "completion_length": 142.75, |
| "epoch": 0.03126706717640634, |
| "grad_norm": 0.02255455031991005, |
| "kl": 0.0006459420278588368, |
| "learning_rate": 4.057894736842106e-06, |
| "loss": 0.0001, |
| "num_tokens": 677400.0, |
| "reward": 0.46562501788139343, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.46562501788139343, |
| "step": 229 |
| }, |
| { |
| "clip_ratio": 0.0006992945709498599, |
| "epoch": 0.031403604587657015, |
| "grad_norm": 0.02236250601708889, |
| "kl": 0.0006349996087919862, |
| "learning_rate": 4.052631578947368e-06, |
| "loss": 0.0, |
| "step": 230 |
| }, |
| { |
| "clip_ratio": 0.0001307531347265467, |
| "epoch": 0.0315401419989077, |
| "grad_norm": 0.02220042608678341, |
| "kl": 0.0006470854705185047, |
| "learning_rate": 4.047368421052632e-06, |
| "loss": 0.0, |
| "step": 231 |
| }, |
| { |
| "clip_ratio": 0.0008540844573872164, |
| "epoch": 0.031676679410158386, |
| "grad_norm": 0.02246800623834133, |
| "kl": 0.0006389342952388688, |
| "learning_rate": 4.042105263157895e-06, |
| "loss": 0.0001, |
| "step": 232 |
| }, |
| { |
| "clip_ratio": 0.00048048709868453443, |
| "completion_length": 149.6875, |
| "epoch": 0.031813216821409064, |
| "grad_norm": 0.02225206233561039, |
| "kl": 0.0006582574633284821, |
| "learning_rate": 4.0368421052631585e-06, |
| "loss": -0.0, |
| "num_tokens": 689414.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 233 |
| }, |
| { |
| "clip_ratio": 0.0016851396212587133, |
| "epoch": 0.03194975423265975, |
| "grad_norm": 0.022733833640813828, |
| "kl": 0.0006728457501594676, |
| "learning_rate": 4.031578947368422e-06, |
| "loss": 0.0001, |
| "step": 234 |
| }, |
| { |
| "clip_ratio": 0.000523468479514122, |
| "epoch": 0.032086291643910435, |
| "grad_norm": 0.022337261587381363, |
| "kl": 0.0006649887154708267, |
| "learning_rate": 4.026315789473684e-06, |
| "loss": 0.0, |
| "step": 235 |
| }, |
| { |
| "clip_ratio": 0.00039966811891645193, |
| "epoch": 0.03222282905516111, |
| "grad_norm": 0.02236855961382389, |
| "kl": 0.0006946580124349566, |
| "learning_rate": 4.021052631578948e-06, |
| "loss": 0.0001, |
| "step": 236 |
| }, |
| { |
| "clip_ratio": 0.0007370143721345812, |
| "completion_length": 139.5, |
| "epoch": 0.0323593664664118, |
| "grad_norm": 0.022790394723415375, |
| "kl": 0.0007473547284462256, |
| "learning_rate": 4.01578947368421e-06, |
| "loss": 0.0002, |
| "num_tokens": 701014.0, |
| "reward": 0.4937499761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.4937499761581421, |
| "step": 237 |
| }, |
| { |
| "clip_ratio": 0.0005428440490504727, |
| "epoch": 0.03249590387766248, |
| "grad_norm": 0.022693805396556854, |
| "kl": 0.0006957066789254895, |
| "learning_rate": 4.010526315789474e-06, |
| "loss": 0.0001, |
| "step": 238 |
| }, |
| { |
| "clip_ratio": 0.0003408734919503331, |
| "epoch": 0.03263244128891316, |
| "grad_norm": 0.02278122492134571, |
| "kl": 0.0007415731188302743, |
| "learning_rate": 4.005263157894737e-06, |
| "loss": 0.0002, |
| "step": 239 |
| }, |
| { |
| "clip_ratio": 0.00013528138515539467, |
| "epoch": 0.03276897870016385, |
| "grad_norm": 0.02245188318192959, |
| "kl": 0.0007022500935818243, |
| "learning_rate": 4.000000000000001e-06, |
| "loss": 0.0001, |
| "step": 240 |
| }, |
| { |
| "clip_ratio": 0.0005109125340823084, |
| "completion_length": 141.28125, |
| "epoch": 0.032905516111414526, |
| "grad_norm": 0.017593976110219955, |
| "kl": 0.000685278452692728, |
| "learning_rate": 3.994736842105264e-06, |
| "loss": 0.0001, |
| "num_tokens": 712323.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 241 |
| }, |
| { |
| "clip_ratio": 0.00013130252773407847, |
| "epoch": 0.03304205352266521, |
| "grad_norm": 0.016243450343608856, |
| "kl": 0.0007128663355615572, |
| "learning_rate": 3.989473684210526e-06, |
| "loss": 0.0001, |
| "step": 242 |
| }, |
| { |
| "clip_ratio": 0.0006084017659304664, |
| "epoch": 0.03317859093391589, |
| "grad_norm": 0.017042577266693115, |
| "kl": 0.0007111189861461753, |
| "learning_rate": 3.98421052631579e-06, |
| "loss": 0.0, |
| "step": 243 |
| }, |
| { |
| "clip_ratio": 0.0006747301958967, |
| "epoch": 0.033315128345166575, |
| "grad_norm": 0.018135398626327515, |
| "kl": 0.000669868842123833, |
| "learning_rate": 3.9789473684210525e-06, |
| "loss": 0.0, |
| "step": 244 |
| }, |
| { |
| "clip_ratio": 0.00022482014901470393, |
| "completion_length": 156.09375, |
| "epoch": 0.03345166575641726, |
| "grad_norm": 0.018865320831537247, |
| "kl": 0.0007178983569247066, |
| "learning_rate": 3.9736842105263165e-06, |
| "loss": -0.0, |
| "num_tokens": 724074.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 245 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03358820316766794, |
| "grad_norm": 0.01888267695903778, |
| "kl": 0.000737229005608242, |
| "learning_rate": 3.968421052631579e-06, |
| "loss": 0.0001, |
| "step": 246 |
| }, |
| { |
| "clip_ratio": 0.00029761905898340046, |
| "epoch": 0.033724740578918624, |
| "grad_norm": 0.018460268154740334, |
| "kl": 0.0007195647185653797, |
| "learning_rate": 3.963157894736843e-06, |
| "loss": 0.0001, |
| "step": 247 |
| }, |
| { |
| "clip_ratio": 0.00040545598312746733, |
| "epoch": 0.03386127799016931, |
| "grad_norm": 0.01854436658322811, |
| "kl": 0.000718526627679239, |
| "learning_rate": 3.957894736842106e-06, |
| "loss": -0.0, |
| "step": 248 |
| }, |
| { |
| "clip_ratio": 0.000364551626262255, |
| "completion_length": 123.5625, |
| "epoch": 0.03399781540141999, |
| "grad_norm": 0.016607927158474922, |
| "kl": 0.0006165570266603027, |
| "learning_rate": 3.952631578947368e-06, |
| "loss": 0.0, |
| "num_tokens": 736048.0, |
| "reward": 0.7750000357627869, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7750000357627869, |
| "step": 249 |
| }, |
| { |
| "clip_ratio": 0.0003149120893795043, |
| "epoch": 0.034134352812670674, |
| "grad_norm": 0.01636321283876896, |
| "kl": 0.0006596833213734499, |
| "learning_rate": 3.947368421052632e-06, |
| "loss": -0.0, |
| "step": 250 |
| }, |
| { |
| "clip_ratio": 0.00033602150506339967, |
| "epoch": 0.03427089022392135, |
| "grad_norm": 0.016457047313451767, |
| "kl": 0.000654415664030239, |
| "learning_rate": 3.942105263157895e-06, |
| "loss": 0.0001, |
| "step": 251 |
| }, |
| { |
| "clip_ratio": 0.00019654087373055518, |
| "epoch": 0.03440742763517204, |
| "grad_norm": 0.01638452522456646, |
| "kl": 0.0006012939916217874, |
| "learning_rate": 3.936842105263159e-06, |
| "loss": 0.0001, |
| "step": 252 |
| }, |
| { |
| "clip_ratio": 0.0009389391634613276, |
| "completion_length": 161.71875, |
| "epoch": 0.03454396504642272, |
| "grad_norm": 0.03075333870947361, |
| "kl": 0.0006442447129302309, |
| "learning_rate": 3.931578947368421e-06, |
| "loss": 0.0002, |
| "num_tokens": 748595.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 253 |
| }, |
| { |
| "clip_ratio": 0.0006016489787725732, |
| "epoch": 0.0346805024576734, |
| "grad_norm": 0.030499013140797615, |
| "kl": 0.0007103143916538102, |
| "learning_rate": 3.926315789473685e-06, |
| "loss": 0.0001, |
| "step": 254 |
| }, |
| { |
| "clip_ratio": 0.0010966659901896492, |
| "epoch": 0.034817039868924086, |
| "grad_norm": 0.030418358743190765, |
| "kl": 0.0007258211571752327, |
| "learning_rate": 3.921052631578947e-06, |
| "loss": 0.0, |
| "step": 255 |
| }, |
| { |
| "clip_ratio": 0.0003524696658132598, |
| "epoch": 0.034953577280174765, |
| "grad_norm": 0.030616672709584236, |
| "kl": 0.0007367220950982301, |
| "learning_rate": 3.9157894736842104e-06, |
| "loss": 0.0001, |
| "step": 256 |
| }, |
| { |
| "clip_ratio": 0.0010683327927836217, |
| "completion_length": 153.0625, |
| "epoch": 0.03509011469142545, |
| "grad_norm": 0.02800179086625576, |
| "kl": 0.0007471022909157909, |
| "learning_rate": 3.9105263157894744e-06, |
| "loss": 0.0002, |
| "num_tokens": 760769.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 257 |
| }, |
| { |
| "clip_ratio": 0.0007546676933998242, |
| "epoch": 0.035226652102676136, |
| "grad_norm": 0.028518524020910263, |
| "kl": 0.000712448805643362, |
| "learning_rate": 3.905263157894737e-06, |
| "loss": 0.0002, |
| "step": 258 |
| }, |
| { |
| "clip_ratio": 0.000800817426352296, |
| "epoch": 0.035363189513926814, |
| "grad_norm": 0.02910081297159195, |
| "kl": 0.0006784590418646985, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": 0.0001, |
| "step": 259 |
| }, |
| { |
| "clip_ratio": 0.0006838897534180433, |
| "epoch": 0.0354997269251775, |
| "grad_norm": 0.027883443981409073, |
| "kl": 0.0007144165310819517, |
| "learning_rate": 3.894736842105263e-06, |
| "loss": -0.0001, |
| "step": 260 |
| }, |
| { |
| "clip_ratio": 0.0007415218569803983, |
| "completion_length": 139.59375, |
| "epoch": 0.035636264336428185, |
| "grad_norm": 0.025706559419631958, |
| "kl": 0.0006818120314164844, |
| "learning_rate": 3.889473684210527e-06, |
| "loss": 0.0001, |
| "num_tokens": 772592.0, |
| "reward": 0.828125, |
| "reward_std": 0.18370190262794495, |
| "rewards/reward_fn": 0.828125, |
| "step": 261 |
| }, |
| { |
| "clip_ratio": 0.0008036253275349736, |
| "epoch": 0.03577280174767886, |
| "grad_norm": 0.02648027054965496, |
| "kl": 0.000685886194332852, |
| "learning_rate": 3.884210526315789e-06, |
| "loss": 0.0, |
| "step": 262 |
| }, |
| { |
| "clip_ratio": 0.0006222750525921583, |
| "epoch": 0.03590933915892955, |
| "grad_norm": 0.025682132691144943, |
| "kl": 0.0006569743923137139, |
| "learning_rate": 3.878947368421053e-06, |
| "loss": -0.0, |
| "step": 263 |
| }, |
| { |
| "clip_ratio": 0.0006222750525921583, |
| "epoch": 0.03604587657018023, |
| "grad_norm": 0.026056313887238503, |
| "kl": 0.0006821834294896689, |
| "learning_rate": 3.873684210526316e-06, |
| "loss": 0.0, |
| "step": 264 |
| }, |
| { |
| "clip_ratio": 0.0006582448550034314, |
| "completion_length": 155.9375, |
| "epoch": 0.03618241398143091, |
| "grad_norm": 0.030329573899507523, |
| "kl": 0.0007156636456784327, |
| "learning_rate": 3.868421052631579e-06, |
| "loss": 0.0002, |
| "num_tokens": 784742.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.298653781414032, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 265 |
| }, |
| { |
| "clip_ratio": 0.0011503307177918032, |
| "epoch": 0.0363189513926816, |
| "grad_norm": 0.031328026205301285, |
| "kl": 0.0007437661415679031, |
| "learning_rate": 3.863157894736843e-06, |
| "loss": 0.0003, |
| "step": 266 |
| }, |
| { |
| "clip_ratio": 0.001414415441104211, |
| "epoch": 0.036455488803932276, |
| "grad_norm": 0.03110205940902233, |
| "kl": 0.0007249777718243422, |
| "learning_rate": 3.857894736842105e-06, |
| "loss": 0.0002, |
| "step": 267 |
| }, |
| { |
| "clip_ratio": 0.0005313157889759168, |
| "epoch": 0.03659202621518296, |
| "grad_norm": 0.031048839911818504, |
| "kl": 0.0006714642913721036, |
| "learning_rate": 3.852631578947369e-06, |
| "loss": 0.0002, |
| "step": 268 |
| }, |
| { |
| "clip_ratio": 0.0004928899579681456, |
| "completion_length": 141.84375, |
| "epoch": 0.03672856362643364, |
| "grad_norm": 0.032659851014614105, |
| "kl": 0.0007797479997861956, |
| "learning_rate": 3.8473684210526316e-06, |
| "loss": 0.0001, |
| "num_tokens": 796197.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.25980761647224426, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 269 |
| }, |
| { |
| "clip_ratio": 0.00028935185400769114, |
| "epoch": 0.036865101037684325, |
| "grad_norm": 0.03203575685620308, |
| "kl": 0.0007469654974556761, |
| "learning_rate": 3.842105263157895e-06, |
| "loss": -0.0, |
| "step": 270 |
| }, |
| { |
| "clip_ratio": 0.0006925335328560323, |
| "epoch": 0.03700163844893501, |
| "grad_norm": 0.03224669024348259, |
| "kl": 0.0008460631906928029, |
| "learning_rate": 3.836842105263158e-06, |
| "loss": 0.0, |
| "step": 271 |
| }, |
| { |
| "clip_ratio": 0.000613619398791343, |
| "epoch": 0.03713817586018569, |
| "grad_norm": 0.03178337216377258, |
| "kl": 0.0008344126890733605, |
| "learning_rate": 3.831578947368421e-06, |
| "loss": 0.0, |
| "step": 272 |
| }, |
| { |
| "clip_ratio": 0.0007658389222342521, |
| "completion_length": 148.75, |
| "epoch": 0.037274713271436374, |
| "grad_norm": 0.02685234323143959, |
| "kl": 0.0008694035859662108, |
| "learning_rate": 3.826315789473685e-06, |
| "loss": -0.0001, |
| "num_tokens": 808537.0, |
| "reward": 0.859375, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.859375, |
| "step": 273 |
| }, |
| { |
| "clip_ratio": 0.00042204382771160454, |
| "epoch": 0.03741125068268706, |
| "grad_norm": 0.02722744457423687, |
| "kl": 0.0008062587003223598, |
| "learning_rate": 3.821052631578947e-06, |
| "loss": -0.0001, |
| "step": 274 |
| }, |
| { |
| "clip_ratio": 0.0006064001499908045, |
| "epoch": 0.03754778809393774, |
| "grad_norm": 0.02703207917511463, |
| "kl": 0.0008390755356231239, |
| "learning_rate": 3.815789473684211e-06, |
| "loss": -0.0002, |
| "step": 275 |
| }, |
| { |
| "clip_ratio": 0.0004554339684545994, |
| "epoch": 0.037684325505188423, |
| "grad_norm": 0.027210967615246773, |
| "kl": 0.0007293312564797816, |
| "learning_rate": 3.810526315789474e-06, |
| "loss": -0.0001, |
| "step": 276 |
| }, |
| { |
| "clip_ratio": 0.001045014927512966, |
| "completion_length": 137.09375, |
| "epoch": 0.0378208629164391, |
| "grad_norm": 0.03316161409020424, |
| "kl": 0.0008686158525961218, |
| "learning_rate": 3.805263157894737e-06, |
| "loss": 0.0001, |
| "num_tokens": 820732.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 277 |
| }, |
| { |
| "clip_ratio": 0.0005361070216167718, |
| "epoch": 0.03795740032768979, |
| "grad_norm": 0.034403275698423386, |
| "kl": 0.0009175811010209145, |
| "learning_rate": 3.8000000000000005e-06, |
| "loss": 0.0003, |
| "step": 278 |
| }, |
| { |
| "clip_ratio": 0.0006925088528078049, |
| "epoch": 0.03809393773894047, |
| "grad_norm": 0.0344231016933918, |
| "kl": 0.0007938581566122593, |
| "learning_rate": 3.794736842105263e-06, |
| "loss": -0.0, |
| "step": 279 |
| }, |
| { |
| "clip_ratio": 0.0013625302235595882, |
| "epoch": 0.03823047515019115, |
| "grad_norm": 0.03481278568506241, |
| "kl": 0.0009434933390366496, |
| "learning_rate": 3.789473684210527e-06, |
| "loss": 0.0001, |
| "step": 280 |
| }, |
| { |
| "clip_ratio": 0.00020424836839083582, |
| "completion_length": 124.59375, |
| "epoch": 0.038367012561441836, |
| "grad_norm": 0.022259853780269623, |
| "kl": 0.0007939861261547776, |
| "learning_rate": 3.7842105263157895e-06, |
| "loss": 0.0002, |
| "num_tokens": 832071.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 281 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.038503549972692515, |
| "grad_norm": 0.022215358912944794, |
| "kl": 0.0008975725677373703, |
| "learning_rate": 3.778947368421053e-06, |
| "loss": 0.0, |
| "step": 282 |
| }, |
| { |
| "clip_ratio": 0.0004173782654106617, |
| "epoch": 0.0386400873839432, |
| "grad_norm": 0.022612985223531723, |
| "kl": 0.0008190222797566094, |
| "learning_rate": 3.773684210526316e-06, |
| "loss": 0.0001, |
| "step": 283 |
| }, |
| { |
| "clip_ratio": 0.00016276042151730508, |
| "epoch": 0.038776624795193886, |
| "grad_norm": 0.022016093134880066, |
| "kl": 0.0008076918084043427, |
| "learning_rate": 3.768421052631579e-06, |
| "loss": 0.0001, |
| "step": 284 |
| }, |
| { |
| "clip_ratio": 0.000884739805769641, |
| "completion_length": 144.40625, |
| "epoch": 0.038913162206444564, |
| "grad_norm": 0.03007262945175171, |
| "kl": 0.0008287996988656232, |
| "learning_rate": 3.7631578947368426e-06, |
| "loss": 0.0, |
| "num_tokens": 844080.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 285 |
| }, |
| { |
| "clip_ratio": 0.0011433004328864627, |
| "epoch": 0.03904969961769525, |
| "grad_norm": 0.029923973605036736, |
| "kl": 0.0008635769727334264, |
| "learning_rate": 3.7578947368421053e-06, |
| "loss": 0.0001, |
| "step": 286 |
| }, |
| { |
| "clip_ratio": 0.0006655217075604014, |
| "epoch": 0.03918623702894593, |
| "grad_norm": 0.029381604865193367, |
| "kl": 0.0008121648797896341, |
| "learning_rate": 3.752631578947369e-06, |
| "loss": -0.0001, |
| "step": 287 |
| }, |
| { |
| "clip_ratio": 0.00125420526455855, |
| "epoch": 0.03932277444019661, |
| "grad_norm": 0.03023548424243927, |
| "kl": 0.0008140054451359902, |
| "learning_rate": 3.7473684210526317e-06, |
| "loss": -0.0001, |
| "step": 288 |
| }, |
| { |
| "clip_ratio": 0.0003085214993916452, |
| "completion_length": 144.75, |
| "epoch": 0.0394593118514473, |
| "grad_norm": 0.02250145748257637, |
| "kl": 0.000704204682733689, |
| "learning_rate": 3.7421052631578953e-06, |
| "loss": -0.0001, |
| "num_tokens": 855800.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 289 |
| }, |
| { |
| "clip_ratio": 0.00027173911803402007, |
| "epoch": 0.03959584926269798, |
| "grad_norm": 0.022790009155869484, |
| "kl": 0.0006996235561018693, |
| "learning_rate": 3.736842105263158e-06, |
| "loss": -0.0001, |
| "step": 290 |
| }, |
| { |
| "clip_ratio": 0.0008939800900407135, |
| "epoch": 0.03973238667394866, |
| "grad_norm": 0.022531475871801376, |
| "kl": 0.0007605676910316106, |
| "learning_rate": 3.7315789473684216e-06, |
| "loss": -0.0002, |
| "step": 291 |
| }, |
| { |
| "clip_ratio": 0.000830608099931851, |
| "epoch": 0.03986892408519935, |
| "grad_norm": 0.022774893790483475, |
| "kl": 0.0007272760003615986, |
| "learning_rate": 3.7263157894736848e-06, |
| "loss": 0.0, |
| "step": 292 |
| }, |
| { |
| "clip_ratio": 0.0004863886861130595, |
| "completion_length": 144.34375, |
| "epoch": 0.040005461496450026, |
| "grad_norm": 0.025127986446022987, |
| "kl": 0.0007745489801891381, |
| "learning_rate": 3.7210526315789475e-06, |
| "loss": 0.0001, |
| "num_tokens": 867691.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2337018847465515, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 293 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04014199890770071, |
| "grad_norm": 0.024640265852212906, |
| "kl": 0.0007439228484145133, |
| "learning_rate": 3.715789473684211e-06, |
| "loss": -0.0001, |
| "step": 294 |
| }, |
| { |
| "clip_ratio": 0.0004863886861130595, |
| "epoch": 0.04027853631895139, |
| "grad_norm": 0.02452578768134117, |
| "kl": 0.0008032110617932631, |
| "learning_rate": 3.710526315789474e-06, |
| "loss": -0.0001, |
| "step": 295 |
| }, |
| { |
| "clip_ratio": 0.00024224806111305952, |
| "epoch": 0.040415073730202075, |
| "grad_norm": 0.02467331476509571, |
| "kl": 0.0007492301938327728, |
| "learning_rate": 3.7052631578947374e-06, |
| "loss": -0.0002, |
| "step": 296 |
| }, |
| { |
| "clip_ratio": 0.000669916596962139, |
| "completion_length": 140.65625, |
| "epoch": 0.04055161114145276, |
| "grad_norm": 0.020856456831097603, |
| "kl": 0.0007392201123366249, |
| "learning_rate": 3.7e-06, |
| "loss": 0.0001, |
| "num_tokens": 879700.0, |
| "reward": 0.46562498807907104, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.46562498807907104, |
| "step": 297 |
| }, |
| { |
| "clip_ratio": 0.00017361111531499773, |
| "epoch": 0.04068814855270344, |
| "grad_norm": 0.02090269699692726, |
| "kl": 0.0008008243175936514, |
| "learning_rate": 3.6947368421052637e-06, |
| "loss": 0.0001, |
| "step": 298 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.040824685963954124, |
| "grad_norm": 0.02103441022336483, |
| "kl": 0.0007869898254284635, |
| "learning_rate": 3.6894736842105265e-06, |
| "loss": 0.0, |
| "step": 299 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0409612233752048, |
| "grad_norm": 0.020737936720252037, |
| "kl": 0.0007604343190905638, |
| "learning_rate": 3.6842105263157896e-06, |
| "loss": -0.0, |
| "step": 300 |
| }, |
| { |
| "clip_ratio": 0.00020559210679493845, |
| "completion_length": 144.25, |
| "epoch": 0.04109776078645549, |
| "grad_norm": 0.02263866737484932, |
| "kl": 0.0008127225319185527, |
| "learning_rate": 3.6789473684210532e-06, |
| "loss": -0.0001, |
| "num_tokens": 891956.0, |
| "reward": 0.859375, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.859375, |
| "step": 301 |
| }, |
| { |
| "clip_ratio": 0.00020559210679493845, |
| "epoch": 0.04123429819770617, |
| "grad_norm": 0.022851375862956047, |
| "kl": 0.0007813003094270243, |
| "learning_rate": 3.673684210526316e-06, |
| "loss": -0.0, |
| "step": 302 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04137083560895685, |
| "grad_norm": 0.02316156215965748, |
| "kl": 0.0007418948507620371, |
| "learning_rate": 3.6684210526315796e-06, |
| "loss": 0.0001, |
| "step": 303 |
| }, |
| { |
| "clip_ratio": 0.00037425151094794273, |
| "epoch": 0.04150737302020754, |
| "grad_norm": 0.02298700250685215, |
| "kl": 0.0008119569065456744, |
| "learning_rate": 3.6631578947368423e-06, |
| "loss": 0.0, |
| "step": 304 |
| }, |
| { |
| "clip_ratio": 0.0004397498705657199, |
| "completion_length": 119.96875, |
| "epoch": 0.04164391043145822, |
| "grad_norm": 0.013564548455178738, |
| "kl": 0.0009123221407207893, |
| "learning_rate": 3.657894736842106e-06, |
| "loss": 0.0001, |
| "num_tokens": 902491.0, |
| "reward": 0.971875011920929, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.971875011920929, |
| "step": 305 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0417804478427089, |
| "grad_norm": 0.01371033862233162, |
| "kl": 0.0009642941181482456, |
| "learning_rate": 3.6526315789473686e-06, |
| "loss": -0.0001, |
| "step": 306 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.041916985253959586, |
| "grad_norm": 0.013809485360980034, |
| "kl": 0.0009494323851413355, |
| "learning_rate": 3.6473684210526318e-06, |
| "loss": 0.0001, |
| "step": 307 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.042053522665210265, |
| "grad_norm": 0.01366579532623291, |
| "kl": 0.0009975432599276246, |
| "learning_rate": 3.642105263157895e-06, |
| "loss": -0.0, |
| "step": 308 |
| }, |
| { |
| "clip_ratio": 0.00017361111531499773, |
| "completion_length": 151.6875, |
| "epoch": 0.04219006007646095, |
| "grad_norm": 0.029042799025774002, |
| "kl": 0.0007244326034197002, |
| "learning_rate": 3.636842105263158e-06, |
| "loss": -0.0, |
| "num_tokens": 914525.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.298653781414032, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 309 |
| }, |
| { |
| "clip_ratio": 0.00020559210679493845, |
| "epoch": 0.042326597487711635, |
| "grad_norm": 0.02877066843211651, |
| "kl": 0.0007629632664247765, |
| "learning_rate": 3.6315789473684217e-06, |
| "loss": 0.0001, |
| "step": 310 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.042463134898962314, |
| "grad_norm": 0.028539150953292847, |
| "kl": 0.0007963754860611516, |
| "learning_rate": 3.6263157894736844e-06, |
| "loss": 0.0001, |
| "step": 311 |
| }, |
| { |
| "clip_ratio": 0.000292056065518409, |
| "epoch": 0.042599672310213, |
| "grad_norm": 0.02861402928829193, |
| "kl": 0.0007653530692550703, |
| "learning_rate": 3.621052631578948e-06, |
| "loss": 0.0001, |
| "step": 312 |
| }, |
| { |
| "clip_ratio": 0.00019171778694726527, |
| "completion_length": 119.59375, |
| "epoch": 0.04273620972146368, |
| "grad_norm": 0.030657904222607613, |
| "kl": 0.0009028592303366167, |
| "learning_rate": 3.6157894736842108e-06, |
| "loss": 0.0002, |
| "num_tokens": 925264.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 313 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04287274713271436, |
| "grad_norm": 0.030080867931246758, |
| "kl": 0.0009077482118300395, |
| "learning_rate": 3.610526315789474e-06, |
| "loss": 0.0001, |
| "step": 314 |
| }, |
| { |
| "clip_ratio": 0.0003787878667935729, |
| "epoch": 0.04300928454396505, |
| "grad_norm": 0.031132614240050316, |
| "kl": 0.0009562106060911901, |
| "learning_rate": 3.605263157894737e-06, |
| "loss": 0.0001, |
| "step": 315 |
| }, |
| { |
| "clip_ratio": 0.0005290678527671844, |
| "epoch": 0.04314582195521573, |
| "grad_norm": 0.03099122643470764, |
| "kl": 0.0010130553719136515, |
| "learning_rate": 3.6000000000000003e-06, |
| "loss": 0.0002, |
| "step": 316 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 135.65625, |
| "epoch": 0.04328235936646641, |
| "grad_norm": 0.02720688469707966, |
| "kl": 0.0009564748033881187, |
| "learning_rate": 3.5947368421052634e-06, |
| "loss": 0.0001, |
| "num_tokens": 937309.0, |
| "reward": 0.859375, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.859375, |
| "step": 317 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0434188967777171, |
| "grad_norm": 0.027184858918190002, |
| "kl": 0.0009709294772619614, |
| "learning_rate": 3.5894736842105266e-06, |
| "loss": 0.0001, |
| "step": 318 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.043555434188967776, |
| "grad_norm": 0.027308201417326927, |
| "kl": 0.0009768028121470707, |
| "learning_rate": 3.58421052631579e-06, |
| "loss": -0.0001, |
| "step": 319 |
| }, |
| { |
| "clip_ratio": 0.00025826445198617876, |
| "epoch": 0.04369197160021846, |
| "grad_norm": 0.026938337832689285, |
| "kl": 0.0008980743559732218, |
| "learning_rate": 3.578947368421053e-06, |
| "loss": -0.0001, |
| "step": 320 |
| }, |
| { |
| "clip_ratio": 0.0005560319623327814, |
| "completion_length": 140.34375, |
| "epoch": 0.04382850901146914, |
| "grad_norm": 0.02959519624710083, |
| "kl": 0.000880028532264987, |
| "learning_rate": 3.5736842105263157e-06, |
| "loss": 0.0001, |
| "num_tokens": 949188.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 321 |
| }, |
| { |
| "clip_ratio": 0.0009927479404723272, |
| "epoch": 0.043965046422719825, |
| "grad_norm": 0.030023330822587013, |
| "kl": 0.0009043698364621378, |
| "learning_rate": 3.5684210526315792e-06, |
| "loss": -0.0001, |
| "step": 322 |
| }, |
| { |
| "clip_ratio": 0.0003880719013977796, |
| "epoch": 0.04410158383397051, |
| "grad_norm": 0.028148531913757324, |
| "kl": 0.0008921725293475902, |
| "learning_rate": 3.5631578947368424e-06, |
| "loss": 0.0002, |
| "step": 323 |
| }, |
| { |
| "clip_ratio": 0.0006740285753039643, |
| "epoch": 0.04423812124522119, |
| "grad_norm": 0.029892824590206146, |
| "kl": 0.0008777467337495182, |
| "learning_rate": 3.5578947368421056e-06, |
| "loss": -0.0, |
| "step": 324 |
| }, |
| { |
| "clip_ratio": 0.00022644927958026528, |
| "completion_length": 108.53125, |
| "epoch": 0.044374658656471874, |
| "grad_norm": 0.03082483820617199, |
| "kl": 0.0009367997481604107, |
| "learning_rate": 3.5526315789473687e-06, |
| "loss": 0.0, |
| "num_tokens": 959709.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 325 |
| }, |
| { |
| "clip_ratio": 0.00022644927958026528, |
| "epoch": 0.04451119606772255, |
| "grad_norm": 0.031293999403715134, |
| "kl": 0.0010181294901485671, |
| "learning_rate": 3.5473684210526323e-06, |
| "loss": -0.0, |
| "step": 326 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04464773347897324, |
| "grad_norm": 0.030545929446816444, |
| "kl": 0.0009729322382554528, |
| "learning_rate": 3.542105263157895e-06, |
| "loss": 0.0001, |
| "step": 327 |
| }, |
| { |
| "clip_ratio": 0.00022644927958026528, |
| "epoch": 0.04478427089022392, |
| "grad_norm": 0.030758768320083618, |
| "kl": 0.0009197474073516787, |
| "learning_rate": 3.536842105263158e-06, |
| "loss": 0.0, |
| "step": 328 |
| }, |
| { |
| "clip_ratio": 0.00018712575547397137, |
| "completion_length": 127.875, |
| "epoch": 0.0449208083014746, |
| "grad_norm": 0.029162487015128136, |
| "kl": 0.0010116572320839623, |
| "learning_rate": 3.5315789473684214e-06, |
| "loss": -0.0, |
| "num_tokens": 970921.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 329 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04505734571272529, |
| "grad_norm": 0.029205741360783577, |
| "kl": 0.0009253943944713683, |
| "learning_rate": 3.5263157894736846e-06, |
| "loss": 0.0003, |
| "step": 330 |
| }, |
| { |
| "clip_ratio": 0.0004011668497696519, |
| "epoch": 0.04519388312397597, |
| "grad_norm": 0.028902918100357056, |
| "kl": 0.0009834289649006678, |
| "learning_rate": 3.5210526315789477e-06, |
| "loss": -0.0, |
| "step": 331 |
| }, |
| { |
| "clip_ratio": 0.00021404109429568052, |
| "epoch": 0.04533042053522665, |
| "grad_norm": 0.02907485142350197, |
| "kl": 0.0008809025821392424, |
| "learning_rate": 3.515789473684211e-06, |
| "loss": 0.0, |
| "step": 332 |
| }, |
| { |
| "clip_ratio": 0.0005798441779916175, |
| "completion_length": 124.75, |
| "epoch": 0.045466957946477336, |
| "grad_norm": 0.03617256507277489, |
| "kl": 0.000798491771092813, |
| "learning_rate": 3.510526315789474e-06, |
| "loss": 0.0001, |
| "num_tokens": 981921.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.298653781414032, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 333 |
| }, |
| { |
| "clip_ratio": 0.00011488970631035045, |
| "epoch": 0.045603495357728015, |
| "grad_norm": 0.03615871071815491, |
| "kl": 0.0008176482906492311, |
| "learning_rate": 3.505263157894737e-06, |
| "loss": -0.0004, |
| "step": 334 |
| }, |
| { |
| "clip_ratio": 0.0014091167831793427, |
| "epoch": 0.0457400327689787, |
| "grad_norm": 0.03517930954694748, |
| "kl": 0.0008173711526069383, |
| "learning_rate": 3.5e-06, |
| "loss": 0.0002, |
| "step": 335 |
| }, |
| { |
| "clip_ratio": 0.0006452707311837003, |
| "epoch": 0.045876570180229385, |
| "grad_norm": 0.03678533807396889, |
| "kl": 0.0008238869195338339, |
| "learning_rate": 3.4947368421052635e-06, |
| "loss": -0.0001, |
| "step": 336 |
| }, |
| { |
| "clip_ratio": 0.0006435871036956087, |
| "completion_length": 171.5625, |
| "epoch": 0.046013107591480064, |
| "grad_norm": 0.030732152983546257, |
| "kl": 0.0008044209635045263, |
| "learning_rate": 3.4894736842105263e-06, |
| "loss": 0.0001, |
| "num_tokens": 994447.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 337 |
| }, |
| { |
| "clip_ratio": 0.0009438273482373916, |
| "epoch": 0.04614964500273075, |
| "grad_norm": 0.030625060200691223, |
| "kl": 0.0007668728567296057, |
| "learning_rate": 3.48421052631579e-06, |
| "loss": -0.0002, |
| "step": 338 |
| }, |
| { |
| "clip_ratio": 0.0010408969974378124, |
| "epoch": 0.04628618241398143, |
| "grad_norm": 0.030618058517575264, |
| "kl": 0.0008538224528820137, |
| "learning_rate": 3.478947368421053e-06, |
| "loss": -0.0001, |
| "step": 339 |
| }, |
| { |
| "clip_ratio": 0.0005348550475900993, |
| "epoch": 0.04642271982523211, |
| "grad_norm": 0.029850170016288757, |
| "kl": 0.0007929099783723359, |
| "learning_rate": 3.473684210526316e-06, |
| "loss": -0.0001, |
| "step": 340 |
| }, |
| { |
| "clip_ratio": 0.0007453141151927412, |
| "completion_length": 136.15625, |
| "epoch": 0.0465592572364828, |
| "grad_norm": 0.028187168762087822, |
| "kl": 0.0007331060778597021, |
| "learning_rate": 3.4684210526315794e-06, |
| "loss": -0.0001, |
| "num_tokens": 1006284.0, |
| "reward": 0.4656250476837158, |
| "reward_std": 0.25110572576522827, |
| "rewards/reward_fn": 0.4656250476837158, |
| "step": 341 |
| }, |
| { |
| "clip_ratio": 0.0011472807818790898, |
| "epoch": 0.04669579464773348, |
| "grad_norm": 0.028124259784817696, |
| "kl": 0.0007809556900610914, |
| "learning_rate": 3.463157894736842e-06, |
| "loss": 0.0001, |
| "step": 342 |
| }, |
| { |
| "clip_ratio": 0.000936449650907889, |
| "epoch": 0.04683233205898416, |
| "grad_norm": 0.028143279254436493, |
| "kl": 0.0007603804115206003, |
| "learning_rate": 3.4578947368421057e-06, |
| "loss": -0.0, |
| "step": 343 |
| }, |
| { |
| "clip_ratio": 0.000640960643067956, |
| "epoch": 0.04696886947023485, |
| "grad_norm": 0.02959413267672062, |
| "kl": 0.0007092921259754803, |
| "learning_rate": 3.4526315789473684e-06, |
| "loss": 0.0002, |
| "step": 344 |
| }, |
| { |
| "clip_ratio": 0.0005915082147112116, |
| "completion_length": 141.1875, |
| "epoch": 0.047105406881485526, |
| "grad_norm": 0.027673540636897087, |
| "kl": 0.0008731992320463178, |
| "learning_rate": 3.447368421052632e-06, |
| "loss": -0.0, |
| "num_tokens": 1017466.0, |
| "reward": 0.71875, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.71875, |
| "step": 345 |
| }, |
| { |
| "clip_ratio": 0.0004701640282291919, |
| "epoch": 0.04724194429273621, |
| "grad_norm": 0.028596054762601852, |
| "kl": 0.0008803572027318296, |
| "learning_rate": 3.4421052631578947e-06, |
| "loss": -0.0001, |
| "step": 346 |
| }, |
| { |
| "clip_ratio": 0.0004612608900060877, |
| "epoch": 0.04737848170398689, |
| "grad_norm": 0.02798176184296608, |
| "kl": 0.0008108743318189227, |
| "learning_rate": 3.4368421052631583e-06, |
| "loss": -0.0003, |
| "step": 347 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.047515019115237575, |
| "grad_norm": 0.028018614277243614, |
| "kl": 0.0007496376688322925, |
| "learning_rate": 3.4315789473684215e-06, |
| "loss": -0.0003, |
| "step": 348 |
| }, |
| { |
| "clip_ratio": 0.00015470296784769744, |
| "completion_length": 131.5, |
| "epoch": 0.04765155652648826, |
| "grad_norm": 0.013035676442086697, |
| "kl": 0.0007108611252988339, |
| "learning_rate": 3.4263157894736842e-06, |
| "loss": -0.0001, |
| "num_tokens": 1028970.0, |
| "reward": 0.4375, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.4375, |
| "step": 349 |
| }, |
| { |
| "clip_ratio": 0.00015470296784769744, |
| "epoch": 0.04778809393773894, |
| "grad_norm": 0.012961960397660732, |
| "kl": 0.0006872391941215028, |
| "learning_rate": 3.421052631578948e-06, |
| "loss": -0.0001, |
| "step": 350 |
| }, |
| { |
| "clip_ratio": 0.00015470296784769744, |
| "epoch": 0.047924631348989624, |
| "grad_norm": 0.013093845918774605, |
| "kl": 0.0007618096824444365, |
| "learning_rate": 3.4157894736842106e-06, |
| "loss": -0.0001, |
| "step": 351 |
| }, |
| { |
| "clip_ratio": 0.0009627791150705889, |
| "epoch": 0.0480611687602403, |
| "grad_norm": 0.013092254288494587, |
| "kl": 0.0007593772188556613, |
| "learning_rate": 3.410526315789474e-06, |
| "loss": 0.0001, |
| "step": 352 |
| }, |
| { |
| "clip_ratio": 0.0012184588267700747, |
| "completion_length": 133.15625, |
| "epoch": 0.04819770617149099, |
| "grad_norm": 0.05045875906944275, |
| "kl": 0.0010233524417344597, |
| "learning_rate": 3.405263157894737e-06, |
| "loss": 0.0002, |
| "num_tokens": 1040631.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.4198557138442993, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 353 |
| }, |
| { |
| "clip_ratio": 0.001344054937362671, |
| "epoch": 0.04833424358274167, |
| "grad_norm": 0.04969387128949165, |
| "kl": 0.0010599575684864249, |
| "learning_rate": 3.4000000000000005e-06, |
| "loss": -0.0001, |
| "step": 354 |
| }, |
| { |
| "clip_ratio": 0.0011714279535226524, |
| "epoch": 0.04847078099399235, |
| "grad_norm": 0.04989272728562355, |
| "kl": 0.0010831583349499851, |
| "learning_rate": 3.3947368421052636e-06, |
| "loss": 0.0002, |
| "step": 355 |
| }, |
| { |
| "clip_ratio": 0.0010523917735554278, |
| "epoch": 0.04860731840524304, |
| "grad_norm": 0.0498301163315773, |
| "kl": 0.0011495640255816397, |
| "learning_rate": 3.3894736842105264e-06, |
| "loss": -0.0001, |
| "step": 356 |
| }, |
| { |
| "clip_ratio": 0.0001707650226308033, |
| "completion_length": 134.96875, |
| "epoch": 0.04874385581649372, |
| "grad_norm": 0.015665635466575623, |
| "kl": 0.0008176299570550327, |
| "learning_rate": 3.38421052631579e-06, |
| "loss": 0.0, |
| "num_tokens": 1052394.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 357 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0488803932277444, |
| "grad_norm": 0.015186646021902561, |
| "kl": 0.0008519297498423839, |
| "learning_rate": 3.3789473684210527e-06, |
| "loss": -0.0001, |
| "step": 358 |
| }, |
| { |
| "clip_ratio": 0.00032552084303461015, |
| "epoch": 0.049016930638995086, |
| "grad_norm": 0.015088633634150028, |
| "kl": 0.0008564612980990205, |
| "learning_rate": 3.3736842105263163e-06, |
| "loss": 0.0001, |
| "step": 359 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.049153468050245765, |
| "grad_norm": 0.0157124362885952, |
| "kl": 0.0008022732190511306, |
| "learning_rate": 3.368421052631579e-06, |
| "loss": -0.0, |
| "step": 360 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "completion_length": 140.40625, |
| "epoch": 0.04929000546149645, |
| "grad_norm": 0.028024472296237946, |
| "kl": 0.0008776496542850509, |
| "learning_rate": 3.3631578947368426e-06, |
| "loss": 0.0001, |
| "num_tokens": 1064123.0, |
| "reward": 0.8031249642372131, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.8031249642372131, |
| "step": 361 |
| }, |
| { |
| "clip_ratio": 0.0001849112450145185, |
| "epoch": 0.049426542872747135, |
| "grad_norm": 0.028005549684166908, |
| "kl": 0.0008063845743890852, |
| "learning_rate": 3.3578947368421054e-06, |
| "loss": 0.0, |
| "step": 362 |
| }, |
| { |
| "clip_ratio": 0.0010166573629248887, |
| "epoch": 0.049563080283997814, |
| "grad_norm": 0.0277395136654377, |
| "kl": 0.0008172210209522746, |
| "learning_rate": 3.3526315789473685e-06, |
| "loss": -0.0, |
| "step": 363 |
| }, |
| { |
| "clip_ratio": 0.0004943171807099134, |
| "epoch": 0.0496996176952485, |
| "grad_norm": 0.028183378279209137, |
| "kl": 0.0008585352416048408, |
| "learning_rate": 3.347368421052632e-06, |
| "loss": 0.0001, |
| "step": 364 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 137.21875, |
| "epoch": 0.04983615510649918, |
| "grad_norm": 0.01484043151140213, |
| "kl": 0.0007672449992242036, |
| "learning_rate": 3.342105263157895e-06, |
| "loss": 0.0, |
| "num_tokens": 1075514.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 365 |
| }, |
| { |
| "clip_ratio": 0.00029761905898340046, |
| "epoch": 0.04997269251774986, |
| "grad_norm": 0.015448699705302715, |
| "kl": 0.0008690163053870492, |
| "learning_rate": 3.3368421052631584e-06, |
| "loss": 0.0001, |
| "step": 366 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05010922992900055, |
| "grad_norm": 0.014827921986579895, |
| "kl": 0.0008811628867988475, |
| "learning_rate": 3.331578947368421e-06, |
| "loss": 0.0001, |
| "step": 367 |
| }, |
| { |
| "clip_ratio": 0.00029761905898340046, |
| "epoch": 0.05024576734025123, |
| "grad_norm": 0.015469317324459553, |
| "kl": 0.0008598208462444745, |
| "learning_rate": 3.3263157894736848e-06, |
| "loss": 0.0001, |
| "step": 368 |
| }, |
| { |
| "clip_ratio": 0.0006039029685780406, |
| "completion_length": 142.125, |
| "epoch": 0.05038230475150191, |
| "grad_norm": 0.05040661618113518, |
| "kl": 0.0017897128709591925, |
| "learning_rate": 3.3210526315789475e-06, |
| "loss": 0.0, |
| "num_tokens": 1087494.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.34620189666748047, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 369 |
| }, |
| { |
| "clip_ratio": 0.001837524599977769, |
| "epoch": 0.0505188421627526, |
| "grad_norm": 0.04978993907570839, |
| "kl": 0.0019511687733029248, |
| "learning_rate": 3.3157894736842107e-06, |
| "loss": -0.0001, |
| "step": 370 |
| }, |
| { |
| "clip_ratio": 0.0009692513267509639, |
| "epoch": 0.050655379574003276, |
| "grad_norm": 0.05005129054188728, |
| "kl": 0.0018768518375509302, |
| "learning_rate": 3.310526315789474e-06, |
| "loss": -0.0002, |
| "step": 371 |
| }, |
| { |
| "clip_ratio": 0.0006632323638768867, |
| "epoch": 0.05079191698525396, |
| "grad_norm": 0.03911221772432327, |
| "kl": 0.0015103784116945462, |
| "learning_rate": 3.305263157894737e-06, |
| "loss": -0.0002, |
| "step": 372 |
| }, |
| { |
| "clip_ratio": 0.0008642399334348738, |
| "completion_length": 123.84375, |
| "epoch": 0.05092845439650464, |
| "grad_norm": 0.03477134928107262, |
| "kl": 0.0009179235403280472, |
| "learning_rate": 3.3000000000000006e-06, |
| "loss": 0.0001, |
| "num_tokens": 1098697.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 373 |
| }, |
| { |
| "clip_ratio": 0.0007545908156316727, |
| "epoch": 0.051064991807755325, |
| "grad_norm": 0.034040454775094986, |
| "kl": 0.0009303928327426547, |
| "learning_rate": 3.2947368421052633e-06, |
| "loss": -0.0002, |
| "step": 374 |
| }, |
| { |
| "clip_ratio": 0.0010670908086467534, |
| "epoch": 0.05120152921900601, |
| "grad_norm": 0.03519129753112793, |
| "kl": 0.0009441823649467551, |
| "learning_rate": 3.289473684210527e-06, |
| "loss": 0.0001, |
| "step": 375 |
| }, |
| { |
| "clip_ratio": 0.0007545908156316727, |
| "epoch": 0.05133806663025669, |
| "grad_norm": 0.03399660438299179, |
| "kl": 0.0009638729961807257, |
| "learning_rate": 3.2842105263157897e-06, |
| "loss": 0.0001, |
| "step": 376 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "completion_length": 150.34375, |
| "epoch": 0.051474604041507374, |
| "grad_norm": 0.02090802974998951, |
| "kl": 0.0008481619370286353, |
| "learning_rate": 3.278947368421053e-06, |
| "loss": -0.0, |
| "num_tokens": 1111616.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 377 |
| }, |
| { |
| "clip_ratio": 0.00041578053787816316, |
| "epoch": 0.05161114145275805, |
| "grad_norm": 0.02162465825676918, |
| "kl": 0.0009244795583072118, |
| "learning_rate": 3.273684210526316e-06, |
| "loss": 0.0, |
| "step": 378 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05174767886400874, |
| "grad_norm": 0.021669728681445122, |
| "kl": 0.0007857876262278296, |
| "learning_rate": 3.268421052631579e-06, |
| "loss": -0.0, |
| "step": 379 |
| }, |
| { |
| "clip_ratio": 0.00026939655072055757, |
| "epoch": 0.05188421627525942, |
| "grad_norm": 0.02159099467098713, |
| "kl": 0.0008994436720968224, |
| "learning_rate": 3.2631578947368423e-06, |
| "loss": -0.0, |
| "step": 380 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 164.21875, |
| "epoch": 0.0520207536865101, |
| "grad_norm": 0.020035242661833763, |
| "kl": 0.0010208635330855031, |
| "learning_rate": 3.2578947368421055e-06, |
| "loss": -0.0, |
| "num_tokens": 1123775.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 381 |
| }, |
| { |
| "clip_ratio": 0.00016191709437407553, |
| "epoch": 0.05215729109776079, |
| "grad_norm": 0.021156245842576027, |
| "kl": 0.0009584235158399679, |
| "learning_rate": 3.252631578947369e-06, |
| "loss": -0.0001, |
| "step": 382 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05229382850901147, |
| "grad_norm": 0.02144774980843067, |
| "kl": 0.0009559597119732643, |
| "learning_rate": 3.247368421052632e-06, |
| "loss": -0.0, |
| "step": 383 |
| }, |
| { |
| "clip_ratio": 0.00016191709437407553, |
| "epoch": 0.05243036592026215, |
| "grad_norm": 0.021347761154174805, |
| "kl": 0.0009219003768521361, |
| "learning_rate": 3.2421052631578945e-06, |
| "loss": 0.0, |
| "step": 384 |
| }, |
| { |
| "clip_ratio": 0.0002741228090599179, |
| "completion_length": 125.1875, |
| "epoch": 0.052566903331512836, |
| "grad_norm": 0.02965986356139183, |
| "kl": 0.0010802704318848555, |
| "learning_rate": 3.236842105263158e-06, |
| "loss": 0.0, |
| "num_tokens": 1134441.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 385 |
| }, |
| { |
| "clip_ratio": 0.0003844313760055229, |
| "epoch": 0.052703440742763515, |
| "grad_norm": 0.029097799211740494, |
| "kl": 0.000968684209510684, |
| "learning_rate": 3.2315789473684213e-06, |
| "loss": 0.0001, |
| "step": 386 |
| }, |
| { |
| "clip_ratio": 0.000584462919505313, |
| "epoch": 0.0528399781540142, |
| "grad_norm": 0.02920384705066681, |
| "kl": 0.0010418808997201268, |
| "learning_rate": 3.2263157894736845e-06, |
| "loss": -0.0001, |
| "step": 387 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.052976515565264885, |
| "grad_norm": 0.03033665381371975, |
| "kl": 0.0010443700157338753, |
| "learning_rate": 3.2210526315789476e-06, |
| "loss": -0.0, |
| "step": 388 |
| }, |
| { |
| "clip_ratio": 0.0006488894578069448, |
| "completion_length": 118.15625, |
| "epoch": 0.053113052976515564, |
| "grad_norm": 0.019158417358994484, |
| "kl": 0.0010731482871051412, |
| "learning_rate": 3.215789473684211e-06, |
| "loss": 0.0002, |
| "num_tokens": 1145598.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 389 |
| }, |
| { |
| "clip_ratio": 0.0002604166802484542, |
| "epoch": 0.05324959038776625, |
| "grad_norm": 0.01972230151295662, |
| "kl": 0.0011826686286440236, |
| "learning_rate": 3.210526315789474e-06, |
| "loss": 0.0001, |
| "step": 390 |
| }, |
| { |
| "clip_ratio": 0.00025826445198617876, |
| "epoch": 0.05338612779901693, |
| "grad_norm": 0.020151875913143158, |
| "kl": 0.001059102758517838, |
| "learning_rate": 3.2052631578947367e-06, |
| "loss": -0.0001, |
| "step": 391 |
| }, |
| { |
| "clip_ratio": 0.00012159533071098849, |
| "epoch": 0.05352266521026761, |
| "grad_norm": 0.020285218954086304, |
| "kl": 0.0010877625118155265, |
| "learning_rate": 3.2000000000000003e-06, |
| "loss": 0.0001, |
| "step": 392 |
| }, |
| { |
| "clip_ratio": 0.00028669723542407155, |
| "completion_length": 124.875, |
| "epoch": 0.0536592026215183, |
| "grad_norm": 0.02960928902029991, |
| "kl": 0.0010169045353904949, |
| "learning_rate": 3.1947368421052634e-06, |
| "loss": -0.0002, |
| "num_tokens": 1156902.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 393 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05379574003276898, |
| "grad_norm": 0.029819682240486145, |
| "kl": 0.00100540832409024, |
| "learning_rate": 3.1894736842105266e-06, |
| "loss": -0.0004, |
| "step": 394 |
| }, |
| { |
| "clip_ratio": 0.00024224806111305952, |
| "epoch": 0.05393227744401966, |
| "grad_norm": 0.029693953692913055, |
| "kl": 0.001026836252094654, |
| "learning_rate": 3.1842105263157898e-06, |
| "loss": -0.0002, |
| "step": 395 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05406881485527035, |
| "grad_norm": 0.029733866453170776, |
| "kl": 0.001003824468170933, |
| "learning_rate": 3.178947368421053e-06, |
| "loss": -0.0001, |
| "step": 396 |
| }, |
| { |
| "clip_ratio": 0.0008818957139737904, |
| "completion_length": 120.90625, |
| "epoch": 0.054205352266521026, |
| "grad_norm": 0.0288943350315094, |
| "kl": 0.0010142269729840336, |
| "learning_rate": 3.173684210526316e-06, |
| "loss": 0.0001, |
| "num_tokens": 1167923.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.25110572576522827, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 397 |
| }, |
| { |
| "clip_ratio": 0.0011852937750518322, |
| "epoch": 0.05434188967777171, |
| "grad_norm": 0.028492823243141174, |
| "kl": 0.001117868141591316, |
| "learning_rate": 3.168421052631579e-06, |
| "loss": 0.0002, |
| "step": 398 |
| }, |
| { |
| "clip_ratio": 0.0006109336100053042, |
| "epoch": 0.05447842708902239, |
| "grad_norm": 0.02876265160739422, |
| "kl": 0.0010261732304570614, |
| "learning_rate": 3.1631578947368424e-06, |
| "loss": 0.0001, |
| "step": 399 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.054614964500273075, |
| "grad_norm": 0.0290256105363369, |
| "kl": 0.0011352793753758306, |
| "learning_rate": 3.157894736842105e-06, |
| "loss": -0.0, |
| "step": 400 |
| }, |
| { |
| "clip_ratio": 0.0006023591413395479, |
| "completion_length": 144.59375, |
| "epoch": 0.05475150191152376, |
| "grad_norm": 0.02488422952592373, |
| "kl": 0.0010054338454210665, |
| "learning_rate": 3.1526315789473688e-06, |
| "loss": 0.0003, |
| "num_tokens": 1179646.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.578125, |
| "step": 401 |
| }, |
| { |
| "clip_ratio": 0.0006849976925877854, |
| "epoch": 0.05488803932277444, |
| "grad_norm": 0.024585776031017303, |
| "kl": 0.0009108762242249213, |
| "learning_rate": 3.147368421052632e-06, |
| "loss": 0.0002, |
| "step": 402 |
| }, |
| { |
| "clip_ratio": 0.0004349976807134226, |
| "epoch": 0.055024576734025124, |
| "grad_norm": 0.02463947981595993, |
| "kl": 0.0009352307270091842, |
| "learning_rate": 3.142105263157895e-06, |
| "loss": 0.0, |
| "step": 403 |
| }, |
| { |
| "clip_ratio": 0.00047781202010810375, |
| "epoch": 0.0551611141452758, |
| "grad_norm": 0.02484274096786976, |
| "kl": 0.000943178461966454, |
| "learning_rate": 3.1368421052631582e-06, |
| "loss": 0.0001, |
| "step": 404 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 132.96875, |
| "epoch": 0.05529765155652649, |
| "grad_norm": 0.027189401909708977, |
| "kl": 0.0009860549935183371, |
| "learning_rate": 3.131578947368421e-06, |
| "loss": -0.0, |
| "num_tokens": 1191045.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.578125, |
| "step": 405 |
| }, |
| { |
| "clip_ratio": 0.00025201612152159214, |
| "epoch": 0.05543418896777717, |
| "grad_norm": 0.027663057669997215, |
| "kl": 0.001040305780406925, |
| "learning_rate": 3.1263157894736846e-06, |
| "loss": -0.0, |
| "step": 406 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05557072637902785, |
| "grad_norm": 0.027513474225997925, |
| "kl": 0.0009971423378374311, |
| "learning_rate": 3.1210526315789473e-06, |
| "loss": 0.0, |
| "step": 407 |
| }, |
| { |
| "clip_ratio": 0.0005750250711571425, |
| "epoch": 0.05570726379027854, |
| "grad_norm": 0.027500582858920097, |
| "kl": 0.0009878080927592237, |
| "learning_rate": 3.115789473684211e-06, |
| "loss": 0.0001, |
| "step": 408 |
| }, |
| { |
| "clip_ratio": 0.00047506178088951856, |
| "completion_length": 128.96875, |
| "epoch": 0.05584380120152922, |
| "grad_norm": 0.02907399833202362, |
| "kl": 0.0009158288730759523, |
| "learning_rate": 3.1105263157894736e-06, |
| "loss": 0.0001, |
| "num_tokens": 1202488.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 409 |
| }, |
| { |
| "clip_ratio": 0.0005465437716338784, |
| "epoch": 0.0559803386127799, |
| "grad_norm": 0.029220519587397575, |
| "kl": 0.0009589706305632717, |
| "learning_rate": 3.1052631578947372e-06, |
| "loss": -0.0001, |
| "step": 410 |
| }, |
| { |
| "clip_ratio": 0.00017458100046496838, |
| "epoch": 0.056116876024030586, |
| "grad_norm": 0.02900395542383194, |
| "kl": 0.0009313819728049566, |
| "learning_rate": 3.1000000000000004e-06, |
| "loss": 0.0, |
| "step": 411 |
| }, |
| { |
| "clip_ratio": 0.00017458100046496838, |
| "epoch": 0.056253413435281265, |
| "grad_norm": 0.029268642887473106, |
| "kl": 0.0009859869569481816, |
| "learning_rate": 3.094736842105263e-06, |
| "loss": -0.0003, |
| "step": 412 |
| }, |
| { |
| "clip_ratio": 0.0002604166802484542, |
| "completion_length": 129.0, |
| "epoch": 0.05638995084653195, |
| "grad_norm": 0.02232382819056511, |
| "kl": 0.0009413343414053088, |
| "learning_rate": 3.0894736842105267e-06, |
| "loss": -0.0, |
| "num_tokens": 1213700.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 413 |
| }, |
| { |
| "clip_ratio": 0.0001755617995513603, |
| "epoch": 0.056526488257782635, |
| "grad_norm": 0.02243160828948021, |
| "kl": 0.0009721652440930484, |
| "learning_rate": 3.0842105263157895e-06, |
| "loss": 0.0003, |
| "step": 414 |
| }, |
| { |
| "clip_ratio": 0.0002604166802484542, |
| "epoch": 0.056663025669033314, |
| "grad_norm": 0.022144054993987083, |
| "kl": 0.000967281979683321, |
| "learning_rate": 3.078947368421053e-06, |
| "loss": -0.0, |
| "step": 415 |
| }, |
| { |
| "clip_ratio": 0.0002604166802484542, |
| "epoch": 0.056799563080284, |
| "grad_norm": 0.02247004024684429, |
| "kl": 0.0010008905883296393, |
| "learning_rate": 3.0736842105263158e-06, |
| "loss": 0.0001, |
| "step": 416 |
| }, |
| { |
| "clip_ratio": 0.0012989779352210462, |
| "completion_length": 126.15625, |
| "epoch": 0.05693610049153468, |
| "grad_norm": 0.03142450377345085, |
| "kl": 0.0011929241682082647, |
| "learning_rate": 3.0684210526315794e-06, |
| "loss": 0.0, |
| "num_tokens": 1225305.0, |
| "reward": 0.578125, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.578125, |
| "step": 417 |
| }, |
| { |
| "clip_ratio": 0.0010220625554211438, |
| "epoch": 0.05707263790278536, |
| "grad_norm": 0.03162077069282532, |
| "kl": 0.001126760105307767, |
| "learning_rate": 3.0631578947368425e-06, |
| "loss": -0.0, |
| "step": 418 |
| }, |
| { |
| "clip_ratio": 0.0013637234515044838, |
| "epoch": 0.05720917531403605, |
| "grad_norm": 0.0330648310482502, |
| "kl": 0.0011995817449133028, |
| "learning_rate": 3.0578947368421053e-06, |
| "loss": 0.0003, |
| "step": 419 |
| }, |
| { |
| "clip_ratio": 0.0012198473705211654, |
| "epoch": 0.05734571272528673, |
| "grad_norm": 0.03194950148463249, |
| "kl": 0.0011162917576257314, |
| "learning_rate": 3.052631578947369e-06, |
| "loss": 0.0001, |
| "step": 420 |
| }, |
| { |
| "clip_ratio": 0.00016891892300918698, |
| "completion_length": 125.84375, |
| "epoch": 0.05748225013653741, |
| "grad_norm": 0.02752695232629776, |
| "kl": 0.0010589144121695426, |
| "learning_rate": 3.0473684210526316e-06, |
| "loss": -0.0001, |
| "num_tokens": 1236240.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 421 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.0576187875477881, |
| "grad_norm": 0.02796095237135887, |
| "kl": 0.0010964719203911955, |
| "learning_rate": 3.042105263157895e-06, |
| "loss": -0.0001, |
| "step": 422 |
| }, |
| { |
| "clip_ratio": 0.0003004807804245502, |
| "epoch": 0.057755324959038776, |
| "grad_norm": 0.028477521613240242, |
| "kl": 0.0011441280676081078, |
| "learning_rate": 3.036842105263158e-06, |
| "loss": -0.0001, |
| "step": 423 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05789186237028946, |
| "grad_norm": 0.027919191867113113, |
| "kl": 0.000996430138911819, |
| "learning_rate": 3.0315789473684215e-06, |
| "loss": -0.0001, |
| "step": 424 |
| }, |
| { |
| "clip_ratio": 0.00034171076549682766, |
| "completion_length": 123.78125, |
| "epoch": 0.05802839978154014, |
| "grad_norm": 0.025030149146914482, |
| "kl": 0.0009267698869734886, |
| "learning_rate": 3.0263157894736843e-06, |
| "loss": -0.0, |
| "num_tokens": 1247061.0, |
| "reward": 0.578125, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.578125, |
| "step": 425 |
| }, |
| { |
| "clip_ratio": 0.00035305789788253605, |
| "epoch": 0.058164937192790825, |
| "grad_norm": 0.02742120996117592, |
| "kl": 0.0009424066829524236, |
| "learning_rate": 3.0210526315789474e-06, |
| "loss": -0.0002, |
| "step": 426 |
| }, |
| { |
| "clip_ratio": 0.00014880952949170023, |
| "epoch": 0.05830147460404151, |
| "grad_norm": 0.024884475395083427, |
| "kl": 0.0008936434551287675, |
| "learning_rate": 3.015789473684211e-06, |
| "loss": 0.0001, |
| "step": 427 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05843801201529219, |
| "grad_norm": 0.024734022095799446, |
| "kl": 0.0008923630566641805, |
| "learning_rate": 3.0105263157894737e-06, |
| "loss": -0.0, |
| "step": 428 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 143.0625, |
| "epoch": 0.058574549426542874, |
| "grad_norm": 0.015420363284647465, |
| "kl": 0.0011353582085575908, |
| "learning_rate": 3.0052631578947373e-06, |
| "loss": -0.0002, |
| "num_tokens": 1258491.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 429 |
| }, |
| { |
| "clip_ratio": 0.0003955696302000433, |
| "epoch": 0.05871108683779355, |
| "grad_norm": 0.015479897148907185, |
| "kl": 0.0011547396525202203, |
| "learning_rate": 3e-06, |
| "loss": -0.0002, |
| "step": 430 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05884762424904424, |
| "grad_norm": 0.015702426433563232, |
| "kl": 0.0011680879488267237, |
| "learning_rate": 2.9947368421052637e-06, |
| "loss": -0.0, |
| "step": 431 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05898416166029492, |
| "grad_norm": 0.015461385250091553, |
| "kl": 0.0011976907471762388, |
| "learning_rate": 2.9894736842105264e-06, |
| "loss": -0.0001, |
| "step": 432 |
| }, |
| { |
| "clip_ratio": 0.0001755617995513603, |
| "completion_length": 134.96875, |
| "epoch": 0.0591206990715456, |
| "grad_norm": 0.03721882402896881, |
| "kl": 0.0010735591249613208, |
| "learning_rate": 2.9842105263157896e-06, |
| "loss": -0.0002, |
| "num_tokens": 1269578.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.298653781414032, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 433 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "epoch": 0.05925723648279629, |
| "grad_norm": 0.03812366724014282, |
| "kl": 0.0011335061217323528, |
| "learning_rate": 2.9789473684210527e-06, |
| "loss": 0.0001, |
| "step": 434 |
| }, |
| { |
| "clip_ratio": 0.0007790682866470888, |
| "epoch": 0.05939377389404697, |
| "grad_norm": 0.037977952510118484, |
| "kl": 0.0010755884868558496, |
| "learning_rate": 2.973684210526316e-06, |
| "loss": -0.0002, |
| "step": 435 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "epoch": 0.05953031130529765, |
| "grad_norm": 0.038097258657217026, |
| "kl": 0.0011848643234770861, |
| "learning_rate": 2.9684210526315795e-06, |
| "loss": -0.0001, |
| "step": 436 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 139.0625, |
| "epoch": 0.059666848716548336, |
| "grad_norm": 0.02606877312064171, |
| "kl": 0.0008522843054379337, |
| "learning_rate": 2.9631578947368422e-06, |
| "loss": 0.0, |
| "num_tokens": 1281764.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 437 |
| }, |
| { |
| "clip_ratio": 0.0004951451264787465, |
| "epoch": 0.059803386127799014, |
| "grad_norm": 0.025950873270630836, |
| "kl": 0.000901157999123825, |
| "learning_rate": 2.957894736842106e-06, |
| "loss": -0.0, |
| "step": 438 |
| }, |
| { |
| "clip_ratio": 0.00032438010384794325, |
| "epoch": 0.0599399235390497, |
| "grad_norm": 0.02585132047533989, |
| "kl": 0.000955841273025726, |
| "learning_rate": 2.9526315789473685e-06, |
| "loss": -0.0, |
| "step": 439 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.060076460950300385, |
| "grad_norm": 0.026470227167010307, |
| "kl": 0.0009173221933451714, |
| "learning_rate": 2.9473684210526317e-06, |
| "loss": -0.0, |
| "step": 440 |
| }, |
| { |
| "clip_ratio": 0.00022321428696159273, |
| "completion_length": 123.15625, |
| "epoch": 0.060212998361551064, |
| "grad_norm": 0.031343188136816025, |
| "kl": 0.0011035336447093869, |
| "learning_rate": 2.942105263157895e-06, |
| "loss": 0.0001, |
| "num_tokens": 1292701.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.25110572576522827, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 441 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06034953577280175, |
| "grad_norm": 0.031478047370910645, |
| "kl": 0.0010842259507626295, |
| "learning_rate": 2.936842105263158e-06, |
| "loss": 0.0001, |
| "step": 442 |
| }, |
| { |
| "clip_ratio": 0.00018939393339678645, |
| "epoch": 0.06048607318405243, |
| "grad_norm": 0.03135223686695099, |
| "kl": 0.0017851412476375117, |
| "learning_rate": 2.931578947368421e-06, |
| "loss": 0.0, |
| "step": 443 |
| }, |
| { |
| "clip_ratio": 0.00022321428696159273, |
| "epoch": 0.06062261059530311, |
| "grad_norm": 0.0316835418343544, |
| "kl": 0.0014097988059802447, |
| "learning_rate": 2.9263157894736844e-06, |
| "loss": 0.0, |
| "step": 444 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 129.5625, |
| "epoch": 0.0607591480065538, |
| "grad_norm": 0.0311806071549654, |
| "kl": 0.0013436454155453248, |
| "learning_rate": 2.921052631578948e-06, |
| "loss": 0.0001, |
| "num_tokens": 1304243.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 445 |
| }, |
| { |
| "clip_ratio": 0.0007211912743514404, |
| "epoch": 0.060895685417804477, |
| "grad_norm": 0.03040701150894165, |
| "kl": 0.0012179117611594847, |
| "learning_rate": 2.9157894736842107e-06, |
| "loss": -0.0002, |
| "step": 446 |
| }, |
| { |
| "clip_ratio": 0.001032393323839642, |
| "epoch": 0.06103222282905516, |
| "grad_norm": 0.030803782865405083, |
| "kl": 0.001290351542593271, |
| "learning_rate": 2.9105263157894743e-06, |
| "loss": -0.0001, |
| "step": 447 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06116876024030585, |
| "grad_norm": 0.030800629407167435, |
| "kl": 0.0013306584751262562, |
| "learning_rate": 2.905263157894737e-06, |
| "loss": -0.0001, |
| "step": 448 |
| }, |
| { |
| "clip_ratio": 0.00028935185400769114, |
| "completion_length": 141.59375, |
| "epoch": 0.061305297651556526, |
| "grad_norm": 0.027338694781064987, |
| "kl": 0.0014368064021255122, |
| "learning_rate": 2.9e-06, |
| "loss": 0.0002, |
| "num_tokens": 1315890.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 449 |
| }, |
| { |
| "clip_ratio": 0.0004810696409549564, |
| "epoch": 0.06144183506280721, |
| "grad_norm": 0.02741483598947525, |
| "kl": 0.0013909679764765315, |
| "learning_rate": 2.8947368421052634e-06, |
| "loss": 0.0001, |
| "step": 450 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06157837247405789, |
| "grad_norm": 0.027010517194867134, |
| "kl": 0.001380715920276998, |
| "learning_rate": 2.8894736842105265e-06, |
| "loss": 0.0, |
| "step": 451 |
| }, |
| { |
| "clip_ratio": 0.0004810696409549564, |
| "epoch": 0.061714909885308575, |
| "grad_norm": 0.027187755331397057, |
| "kl": 0.0014208953525667312, |
| "learning_rate": 2.88421052631579e-06, |
| "loss": -0.0, |
| "step": 452 |
| }, |
| { |
| "clip_ratio": 0.0008928571478463709, |
| "completion_length": 145.4375, |
| "epoch": 0.06185144729655926, |
| "grad_norm": 0.027749359607696533, |
| "kl": 0.001469686958444072, |
| "learning_rate": 2.878947368421053e-06, |
| "loss": 0.0001, |
| "num_tokens": 1327856.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 453 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06198798470780994, |
| "grad_norm": 0.0267344918102026, |
| "kl": 0.001519542402093066, |
| "learning_rate": 2.8736842105263164e-06, |
| "loss": 0.0, |
| "step": 454 |
| }, |
| { |
| "clip_ratio": 0.0003810975467786193, |
| "epoch": 0.062124522119060624, |
| "grad_norm": 0.027275780215859413, |
| "kl": 0.001477261032050592, |
| "learning_rate": 2.868421052631579e-06, |
| "loss": 0.0001, |
| "step": 455 |
| }, |
| { |
| "clip_ratio": 0.00044642857392318547, |
| "epoch": 0.0622610595303113, |
| "grad_norm": 0.027177719399333, |
| "kl": 0.0014757139615539927, |
| "learning_rate": 2.8631578947368423e-06, |
| "loss": 0.0, |
| "step": 456 |
| }, |
| { |
| "clip_ratio": 0.0011708111996995285, |
| "completion_length": 145.4375, |
| "epoch": 0.06239759694156199, |
| "grad_norm": 0.03548799455165863, |
| "kl": 0.0013813442674290854, |
| "learning_rate": 2.8578947368421055e-06, |
| "loss": 0.0002, |
| "num_tokens": 1339522.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.30735570192337036, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 457 |
| }, |
| { |
| "clip_ratio": 0.0011642339522950351, |
| "epoch": 0.06253413435281267, |
| "grad_norm": 0.03586866334080696, |
| "kl": 0.0013935480292275315, |
| "learning_rate": 2.8526315789473687e-06, |
| "loss": 0.0002, |
| "step": 458 |
| }, |
| { |
| "clip_ratio": 0.0009245706896763295, |
| "epoch": 0.06267067176406335, |
| "grad_norm": 0.035171736031770706, |
| "kl": 0.0013296840588736814, |
| "learning_rate": 2.847368421052632e-06, |
| "loss": -0.0001, |
| "step": 459 |
| }, |
| { |
| "clip_ratio": 0.0005984370218357071, |
| "epoch": 0.06280720917531403, |
| "grad_norm": 0.03586185351014137, |
| "kl": 0.0013636875646625413, |
| "learning_rate": 2.842105263157895e-06, |
| "loss": 0.0001, |
| "step": 460 |
| }, |
| { |
| "clip_ratio": 0.00046950820251367986, |
| "completion_length": 118.875, |
| "epoch": 0.06294374658656472, |
| "grad_norm": 0.03280496224761009, |
| "kl": 0.0013567173700721469, |
| "learning_rate": 2.8368421052631586e-06, |
| "loss": 0.0002, |
| "num_tokens": 1350862.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 461 |
| }, |
| { |
| "clip_ratio": 0.0004111842135898769, |
| "epoch": 0.0630802839978154, |
| "grad_norm": 0.03311141952872276, |
| "kl": 0.0014376636845554458, |
| "learning_rate": 2.8315789473684213e-06, |
| "loss": 0.0003, |
| "step": 462 |
| }, |
| { |
| "clip_ratio": 0.00044646734022535384, |
| "epoch": 0.06321682140906608, |
| "grad_norm": 0.03446916490793228, |
| "kl": 0.0015070214722072706, |
| "learning_rate": 2.826315789473684e-06, |
| "loss": 0.0003, |
| "step": 463 |
| }, |
| { |
| "clip_ratio": 0.0007496662874473259, |
| "epoch": 0.06335335882031677, |
| "grad_norm": 0.03369483724236488, |
| "kl": 0.001399203034452512, |
| "learning_rate": 2.8210526315789476e-06, |
| "loss": 0.0001, |
| "step": 464 |
| }, |
| { |
| "clip_ratio": 0.0006772563501726836, |
| "completion_length": 126.5625, |
| "epoch": 0.06348989623156745, |
| "grad_norm": 0.034894075244665146, |
| "kl": 0.0011480197363198386, |
| "learning_rate": 2.815789473684211e-06, |
| "loss": 0.0001, |
| "num_tokens": 1361844.0, |
| "reward": 0.6875, |
| "reward_std": 0.3049038052558899, |
| "rewards/reward_fn": 0.6875, |
| "step": 465 |
| }, |
| { |
| "clip_ratio": 0.0009165278461296111, |
| "epoch": 0.06362643364281813, |
| "grad_norm": 0.03461375832557678, |
| "kl": 0.0012377909497445216, |
| "learning_rate": 2.810526315789474e-06, |
| "loss": 0.0, |
| "step": 466 |
| }, |
| { |
| "clip_ratio": 0.0010300020658178255, |
| "epoch": 0.06376297105406882, |
| "grad_norm": 0.03713339567184448, |
| "kl": 0.0013127127203915734, |
| "learning_rate": 2.805263157894737e-06, |
| "loss": 0.0001, |
| "step": 467 |
| }, |
| { |
| "clip_ratio": 0.00047134239866863936, |
| "epoch": 0.0638995084653195, |
| "grad_norm": 0.03711392730474472, |
| "kl": 0.001211642022099113, |
| "learning_rate": 2.8000000000000003e-06, |
| "loss": 0.0002, |
| "step": 468 |
| }, |
| { |
| "clip_ratio": 0.0008621815213700756, |
| "completion_length": 162.28125, |
| "epoch": 0.06403604587657018, |
| "grad_norm": 0.026019882410764694, |
| "kl": 0.0012119517618884856, |
| "learning_rate": 2.7947368421052635e-06, |
| "loss": 0.0001, |
| "num_tokens": 1374489.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 469 |
| }, |
| { |
| "clip_ratio": 0.00040302579873241484, |
| "epoch": 0.06417258328782087, |
| "grad_norm": 0.025591466575860977, |
| "kl": 0.0012105955256629386, |
| "learning_rate": 2.789473684210526e-06, |
| "loss": 0.0, |
| "step": 470 |
| }, |
| { |
| "clip_ratio": 0.0014681293978355825, |
| "epoch": 0.06430912069907155, |
| "grad_norm": 0.025961099192500114, |
| "kl": 0.001206063278914371, |
| "learning_rate": 2.7842105263157898e-06, |
| "loss": 0.0002, |
| "step": 471 |
| }, |
| { |
| "clip_ratio": 0.0008783431985648349, |
| "epoch": 0.06444565811032223, |
| "grad_norm": 0.026147864758968353, |
| "kl": 0.0011393453842174495, |
| "learning_rate": 2.7789473684210525e-06, |
| "loss": 0.0001, |
| "step": 472 |
| }, |
| { |
| "clip_ratio": 0.00012755101488437504, |
| "completion_length": 129.78125, |
| "epoch": 0.0645821955215729, |
| "grad_norm": 0.03525552526116371, |
| "kl": 0.001519918667327147, |
| "learning_rate": 2.773684210526316e-06, |
| "loss": -0.0001, |
| "num_tokens": 1385870.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 473 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0647187329328236, |
| "grad_norm": 0.03530130535364151, |
| "kl": 0.0015549474192084745, |
| "learning_rate": 2.7684210526315793e-06, |
| "loss": 0.0002, |
| "step": 474 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "epoch": 0.06485527034407428, |
| "grad_norm": 0.03444594517350197, |
| "kl": 0.0015299975057132542, |
| "learning_rate": 2.7631578947368424e-06, |
| "loss": 0.0, |
| "step": 475 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06499180775532495, |
| "grad_norm": 0.03540601208806038, |
| "kl": 0.0014829309839115012, |
| "learning_rate": 2.7578947368421056e-06, |
| "loss": -0.0002, |
| "step": 476 |
| }, |
| { |
| "completion_length": 139.4375, |
| "epoch": 0.06512834516657565, |
| "grad_norm": 0.0, |
| "learning_rate": 2.7526315789473683e-06, |
| "loss": 0.0, |
| "num_tokens": 1398096.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.0, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 477 |
| }, |
| { |
| "epoch": 0.06526488257782632, |
| "grad_norm": 0.0, |
| "learning_rate": 2.747368421052632e-06, |
| "loss": 0.0, |
| "step": 478 |
| }, |
| { |
| "epoch": 0.065401419989077, |
| "grad_norm": 0.0, |
| "learning_rate": 2.7421052631578947e-06, |
| "loss": 0.0, |
| "step": 479 |
| }, |
| { |
| "epoch": 0.0655379574003277, |
| "grad_norm": 0.0, |
| "learning_rate": 2.7368421052631583e-06, |
| "loss": 0.0, |
| "step": 480 |
| }, |
| { |
| "clip_ratio": 0.00048363096721004695, |
| "completion_length": 139.28125, |
| "epoch": 0.06567449481157837, |
| "grad_norm": 0.03068307414650917, |
| "kl": 0.0013596344333564048, |
| "learning_rate": 2.7315789473684214e-06, |
| "loss": 0.0001, |
| "num_tokens": 1410073.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 481 |
| }, |
| { |
| "clip_ratio": 0.0004263965238351375, |
| "epoch": 0.06581103222282905, |
| "grad_norm": 0.03083832748234272, |
| "kl": 0.0012160772521383478, |
| "learning_rate": 2.7263157894736846e-06, |
| "loss": 0.0002, |
| "step": 482 |
| }, |
| { |
| "clip_ratio": 0.0007327690836973488, |
| "epoch": 0.06594756963407974, |
| "grad_norm": 0.03032069467008114, |
| "kl": 0.0012884464822491282, |
| "learning_rate": 2.7210526315789478e-06, |
| "loss": 0.0, |
| "step": 483 |
| }, |
| { |
| "clip_ratio": 0.0004923844680888578, |
| "epoch": 0.06608410704533042, |
| "grad_norm": 0.030407611280679703, |
| "kl": 0.0013384391750150826, |
| "learning_rate": 2.7157894736842105e-06, |
| "loss": -0.0002, |
| "step": 484 |
| }, |
| { |
| "clip_ratio": 0.0007657096430193633, |
| "completion_length": 120.59375, |
| "epoch": 0.0662206444565811, |
| "grad_norm": 0.043436191976070404, |
| "kl": 0.0014728355108672986, |
| "learning_rate": 2.710526315789474e-06, |
| "loss": 0.0001, |
| "num_tokens": 1420660.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.30735570192337036, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 485 |
| }, |
| { |
| "clip_ratio": 0.0010727702174335718, |
| "epoch": 0.06635718186783178, |
| "grad_norm": 0.044896580278873444, |
| "kl": 0.0015333851679315558, |
| "learning_rate": 2.705263157894737e-06, |
| "loss": 0.0006, |
| "step": 486 |
| }, |
| { |
| "clip_ratio": 0.0008705731888767332, |
| "epoch": 0.06649371927908247, |
| "grad_norm": 0.04429871216416359, |
| "kl": 0.0014375828959600767, |
| "learning_rate": 2.7000000000000004e-06, |
| "loss": 0.0003, |
| "step": 487 |
| }, |
| { |
| "clip_ratio": 0.0010845871875062585, |
| "epoch": 0.06663025669033315, |
| "grad_norm": 0.04436115548014641, |
| "kl": 0.0015077010029926896, |
| "learning_rate": 2.694736842105263e-06, |
| "loss": 0.0001, |
| "step": 488 |
| }, |
| { |
| "clip_ratio": 0.00015703517419751734, |
| "completion_length": 138.9375, |
| "epoch": 0.06676679410158383, |
| "grad_norm": 0.025293614715337753, |
| "kl": 0.0016928213699429762, |
| "learning_rate": 2.6894736842105267e-06, |
| "loss": -0.0001, |
| "num_tokens": 1431886.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.578125, |
| "step": 489 |
| }, |
| { |
| "clip_ratio": 0.00044642855937127024, |
| "epoch": 0.06690333151283452, |
| "grad_norm": 0.025101078674197197, |
| "kl": 0.001892799466077122, |
| "learning_rate": 2.68421052631579e-06, |
| "loss": -0.0002, |
| "step": 490 |
| }, |
| { |
| "clip_ratio": 0.0003188775444868952, |
| "epoch": 0.0670398689240852, |
| "grad_norm": 0.025167712941765785, |
| "kl": 0.001736639860610012, |
| "learning_rate": 2.6789473684210526e-06, |
| "loss": -0.0, |
| "step": 491 |
| }, |
| { |
| "clip_ratio": 0.0009300961974076927, |
| "epoch": 0.06717640633533588, |
| "grad_norm": 0.025257227942347527, |
| "kl": 0.0017978880205191672, |
| "learning_rate": 2.6736842105263162e-06, |
| "loss": -0.0001, |
| "step": 492 |
| }, |
| { |
| "clip_ratio": 0.0003878545976476744, |
| "completion_length": 147.4375, |
| "epoch": 0.06731294374658657, |
| "grad_norm": 0.022938350215554237, |
| "kl": 0.0015158076530497056, |
| "learning_rate": 2.668421052631579e-06, |
| "loss": -0.0, |
| "num_tokens": 1444020.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 493 |
| }, |
| { |
| "clip_ratio": 0.0008949182229116559, |
| "epoch": 0.06744948115783725, |
| "grad_norm": 0.023230843245983124, |
| "kl": 0.0014488295801129425, |
| "learning_rate": 2.6631578947368426e-06, |
| "loss": 0.0, |
| "step": 494 |
| }, |
| { |
| "clip_ratio": 0.0007672396022826433, |
| "epoch": 0.06758601856908793, |
| "grad_norm": 0.023685956373810768, |
| "kl": 0.0014960741591494298, |
| "learning_rate": 2.6578947368421053e-06, |
| "loss": 0.0001, |
| "step": 495 |
| }, |
| { |
| "clip_ratio": 0.0004285848408471793, |
| "epoch": 0.06772255598033862, |
| "grad_norm": 0.022663943469524384, |
| "kl": 0.0014857233782095136, |
| "learning_rate": 2.652631578947369e-06, |
| "loss": -0.0001, |
| "step": 496 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 133.8125, |
| "epoch": 0.0678590933915893, |
| "grad_norm": 0.023737484589219093, |
| "kl": 0.0015226335908664623, |
| "learning_rate": 2.6473684210526316e-06, |
| "loss": 0.0, |
| "num_tokens": 1455314.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 497 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06799563080283998, |
| "grad_norm": 0.023308604955673218, |
| "kl": 0.0014484160819847602, |
| "learning_rate": 2.6421052631578948e-06, |
| "loss": 0.0001, |
| "step": 498 |
| }, |
| { |
| "clip_ratio": 0.00022163119865581393, |
| "epoch": 0.06813216821409065, |
| "grad_norm": 0.02275722846388817, |
| "kl": 0.0015247040919348365, |
| "learning_rate": 2.6368421052631584e-06, |
| "loss": -0.0, |
| "step": 499 |
| }, |
| { |
| "clip_ratio": 0.00018939393339678645, |
| "epoch": 0.06826870562534135, |
| "grad_norm": 0.024789828807115555, |
| "kl": 0.0015402598492073594, |
| "learning_rate": 2.631578947368421e-06, |
| "loss": 0.0001, |
| "step": 500 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 133.34375, |
| "epoch": 0.06840524303659203, |
| "grad_norm": 0.029355965554714203, |
| "kl": 0.0014410110106837237, |
| "learning_rate": 2.6263157894736847e-06, |
| "loss": -0.0001, |
| "num_tokens": 1466205.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 501 |
| }, |
| { |
| "clip_ratio": 0.0008779402705840766, |
| "epoch": 0.0685417804478427, |
| "grad_norm": 0.02950431779026985, |
| "kl": 0.0014032465078344103, |
| "learning_rate": 2.6210526315789474e-06, |
| "loss": 0.0001, |
| "step": 502 |
| }, |
| { |
| "clip_ratio": 0.0004614599165506661, |
| "epoch": 0.0686783178590934, |
| "grad_norm": 0.02926722541451454, |
| "kl": 0.0014136335121293087, |
| "learning_rate": 2.615789473684211e-06, |
| "loss": 0.0001, |
| "step": 503 |
| }, |
| { |
| "clip_ratio": 0.00046644278336316347, |
| "epoch": 0.06881485527034407, |
| "grad_norm": 0.02934475615620613, |
| "kl": 0.0013926027413617703, |
| "learning_rate": 2.6105263157894738e-06, |
| "loss": -0.0001, |
| "step": 504 |
| }, |
| { |
| "clip_ratio": 0.00032552084303461015, |
| "completion_length": 117.78125, |
| "epoch": 0.06895139268159475, |
| "grad_norm": 0.0215784702450037, |
| "kl": 0.0014604836833314039, |
| "learning_rate": 2.605263157894737e-06, |
| "loss": 0.0, |
| "num_tokens": 1476486.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 505 |
| }, |
| { |
| "clip_ratio": 0.00032552084303461015, |
| "epoch": 0.06908793009284545, |
| "grad_norm": 0.02178478240966797, |
| "kl": 0.0016060788602771936, |
| "learning_rate": 2.6e-06, |
| "loss": -0.0001, |
| "step": 506 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06922446750409612, |
| "grad_norm": 0.021680708974599838, |
| "kl": 0.0015948796844895696, |
| "learning_rate": 2.5947368421052633e-06, |
| "loss": 0.0001, |
| "step": 507 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0693610049153468, |
| "grad_norm": 0.021806875243782997, |
| "kl": 0.001626272858629818, |
| "learning_rate": 2.589473684210527e-06, |
| "loss": -0.0001, |
| "step": 508 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 128.96875, |
| "epoch": 0.0694975423265975, |
| "grad_norm": 0.025997139513492584, |
| "kl": 0.0012444071562640602, |
| "learning_rate": 2.5842105263157896e-06, |
| "loss": -0.0001, |
| "num_tokens": 1487469.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 509 |
| }, |
| { |
| "clip_ratio": 0.000666738415020518, |
| "epoch": 0.06963407973784817, |
| "grad_norm": 0.026637002825737, |
| "kl": 0.0013541714697566931, |
| "learning_rate": 2.578947368421053e-06, |
| "loss": -0.0001, |
| "step": 510 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06977061714909885, |
| "grad_norm": 0.026402544230222702, |
| "kl": 0.0012964879524588468, |
| "learning_rate": 2.573684210526316e-06, |
| "loss": -0.0, |
| "step": 511 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06990715456034953, |
| "grad_norm": 0.0267286766320467, |
| "kl": 0.0014071960695218877, |
| "learning_rate": 2.568421052631579e-06, |
| "loss": -0.0, |
| "step": 512 |
| }, |
| { |
| "clip_ratio": 0.0003955851716455072, |
| "completion_length": 118.46875, |
| "epoch": 0.07004369197160022, |
| "grad_norm": 0.03863956406712532, |
| "kl": 0.001288709736400051, |
| "learning_rate": 2.5631578947368422e-06, |
| "loss": -0.0003, |
| "num_tokens": 1498808.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.30735570192337036, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 513 |
| }, |
| { |
| "clip_ratio": 0.00048096767568495125, |
| "epoch": 0.0701802293828509, |
| "grad_norm": 0.03899668529629707, |
| "kl": 0.0013409877155936556, |
| "learning_rate": 2.5578947368421054e-06, |
| "loss": -0.0, |
| "step": 514 |
| }, |
| { |
| "clip_ratio": 0.0004269125493010506, |
| "epoch": 0.07031676679410158, |
| "grad_norm": 0.03946976363658905, |
| "kl": 0.0014272014959715307, |
| "learning_rate": 2.552631578947369e-06, |
| "loss": -0.0001, |
| "step": 515 |
| }, |
| { |
| "clip_ratio": 0.0005278866447042674, |
| "epoch": 0.07045330420535227, |
| "grad_norm": 0.03886709362268448, |
| "kl": 0.0013788384367217077, |
| "learning_rate": 2.5473684210526317e-06, |
| "loss": -0.0, |
| "step": 516 |
| }, |
| { |
| "clip_ratio": 0.0007051962893456221, |
| "completion_length": 137.40625, |
| "epoch": 0.07058984161660295, |
| "grad_norm": 0.030156614258885384, |
| "kl": 0.0012403783885019948, |
| "learning_rate": 2.5421052631578953e-06, |
| "loss": -0.0002, |
| "num_tokens": 1510737.0, |
| "reward": 0.4937500059604645, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.4937500059604645, |
| "step": 517 |
| }, |
| { |
| "clip_ratio": 0.0010378391889389604, |
| "epoch": 0.07072637902785363, |
| "grad_norm": 0.030437057837843895, |
| "kl": 0.0012195480640002643, |
| "learning_rate": 2.536842105263158e-06, |
| "loss": 0.0001, |
| "step": 518 |
| }, |
| { |
| "clip_ratio": 0.0005122950533404946, |
| "epoch": 0.07086291643910432, |
| "grad_norm": 0.030496113002300262, |
| "kl": 0.001190645452879835, |
| "learning_rate": 2.5315789473684212e-06, |
| "loss": -0.0002, |
| "step": 519 |
| }, |
| { |
| "clip_ratio": 0.0006950435927137733, |
| "epoch": 0.070999453850355, |
| "grad_norm": 0.02995339408516884, |
| "kl": 0.0010328257139917696, |
| "learning_rate": 2.5263157894736844e-06, |
| "loss": -0.0002, |
| "step": 520 |
| }, |
| { |
| "clip_ratio": 0.00037165620597079396, |
| "completion_length": 130.6875, |
| "epoch": 0.07113599126160568, |
| "grad_norm": 0.027491774410009384, |
| "kl": 0.0015749909580335952, |
| "learning_rate": 2.5210526315789475e-06, |
| "loss": 0.0003, |
| "num_tokens": 1521387.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 521 |
| }, |
| { |
| "clip_ratio": 0.0002460629912093282, |
| "epoch": 0.07127252867285637, |
| "grad_norm": 0.02750663459300995, |
| "kl": 0.0013783774675175664, |
| "learning_rate": 2.5157894736842107e-06, |
| "loss": 0.0002, |
| "step": 522 |
| }, |
| { |
| "clip_ratio": 0.0005790924478787929, |
| "epoch": 0.07140906608410705, |
| "grad_norm": 0.027644384652376175, |
| "kl": 0.0013931172179582063, |
| "learning_rate": 2.510526315789474e-06, |
| "loss": 0.0003, |
| "step": 523 |
| }, |
| { |
| "clip_ratio": 0.00037165620597079396, |
| "epoch": 0.07154560349535773, |
| "grad_norm": 0.02771710976958275, |
| "kl": 0.001518670924269827, |
| "learning_rate": 2.5052631578947375e-06, |
| "loss": 0.0002, |
| "step": 524 |
| }, |
| { |
| "completion_length": 133.84375, |
| "epoch": 0.0716821409066084, |
| "grad_norm": 0.0, |
| "learning_rate": 2.5e-06, |
| "loss": 0.0, |
| "num_tokens": 1532574.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.0, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 525 |
| }, |
| { |
| "epoch": 0.0718186783178591, |
| "grad_norm": 0.0, |
| "learning_rate": 2.4947368421052634e-06, |
| "loss": 0.0, |
| "step": 526 |
| }, |
| { |
| "epoch": 0.07195521572910978, |
| "grad_norm": 0.0, |
| "learning_rate": 2.4894736842105265e-06, |
| "loss": 0.0, |
| "step": 527 |
| }, |
| { |
| "epoch": 0.07209175314036045, |
| "grad_norm": 0.0, |
| "learning_rate": 2.4842105263157897e-06, |
| "loss": 0.0, |
| "step": 528 |
| }, |
| { |
| "clip_ratio": 0.0002982954465551302, |
| "completion_length": 142.46875, |
| "epoch": 0.07222829055161115, |
| "grad_norm": 0.029288657009601593, |
| "kl": 0.0009811991058086278, |
| "learning_rate": 2.478947368421053e-06, |
| "loss": 0.0003, |
| "num_tokens": 1544593.0, |
| "reward": 0.7718750238418579, |
| "reward_std": 0.2962018847465515, |
| "rewards/reward_fn": 0.7718750238418579, |
| "step": 529 |
| }, |
| { |
| "clip_ratio": 0.0008637555874884129, |
| "epoch": 0.07236482796286182, |
| "grad_norm": 0.029271140694618225, |
| "kl": 0.0010114734732269426, |
| "learning_rate": 2.473684210526316e-06, |
| "loss": 0.0003, |
| "step": 530 |
| }, |
| { |
| "clip_ratio": 0.0008495510555803776, |
| "epoch": 0.0725013653741125, |
| "grad_norm": 0.028701355680823326, |
| "kl": 0.0010413858981337398, |
| "learning_rate": 2.468421052631579e-06, |
| "loss": 0.0002, |
| "step": 531 |
| }, |
| { |
| "clip_ratio": 0.0005510496121132746, |
| "epoch": 0.0726379027853632, |
| "grad_norm": 0.02870851755142212, |
| "kl": 0.0010655641935954918, |
| "learning_rate": 2.4631578947368424e-06, |
| "loss": 0.0004, |
| "step": 532 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 131.78125, |
| "epoch": 0.07277444019661387, |
| "grad_norm": 0.024263199418783188, |
| "kl": 0.0011644043825072004, |
| "learning_rate": 2.4578947368421055e-06, |
| "loss": -0.0002, |
| "num_tokens": 1556426.0, |
| "reward": 0.578125, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.578125, |
| "step": 533 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07291097760786455, |
| "grad_norm": 0.023698801174759865, |
| "kl": 0.001130858380747668, |
| "learning_rate": 2.4526315789473687e-06, |
| "loss": -0.0002, |
| "step": 534 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "epoch": 0.07304751501911524, |
| "grad_norm": 0.024516500532627106, |
| "kl": 0.001214617892401293, |
| "learning_rate": 2.447368421052632e-06, |
| "loss": -0.0, |
| "step": 535 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "epoch": 0.07318405243036592, |
| "grad_norm": 0.0237928107380867, |
| "kl": 0.0012256574964339961, |
| "learning_rate": 2.442105263157895e-06, |
| "loss": -0.0002, |
| "step": 536 |
| }, |
| { |
| "clip_ratio": 0.0012445505853975192, |
| "completion_length": 140.96875, |
| "epoch": 0.0733205898416166, |
| "grad_norm": 0.034096427261829376, |
| "kl": 0.001380213063384872, |
| "learning_rate": 2.436842105263158e-06, |
| "loss": -0.0001, |
| "num_tokens": 1568913.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.35490381717681885, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 537 |
| }, |
| { |
| "clip_ratio": 0.0008802999946055934, |
| "epoch": 0.07345712725286728, |
| "grad_norm": 0.033610839396715164, |
| "kl": 0.0014004747145008878, |
| "learning_rate": 2.4315789473684213e-06, |
| "loss": -0.0001, |
| "step": 538 |
| }, |
| { |
| "clip_ratio": 0.00060980801936239, |
| "epoch": 0.07359366466411797, |
| "grad_norm": 0.03407885134220123, |
| "kl": 0.0013439315152936615, |
| "learning_rate": 2.4263157894736845e-06, |
| "loss": -0.0001, |
| "step": 539 |
| }, |
| { |
| "clip_ratio": 0.0004953578027198091, |
| "epoch": 0.07373020207536865, |
| "grad_norm": 0.033993177115917206, |
| "kl": 0.0013473886228894116, |
| "learning_rate": 2.4210526315789477e-06, |
| "loss": -0.0, |
| "step": 540 |
| }, |
| { |
| "clip_ratio": 0.001183721236884594, |
| "completion_length": 155.21875, |
| "epoch": 0.07386673948661933, |
| "grad_norm": 0.0310914795845747, |
| "kl": 0.0014184249976096908, |
| "learning_rate": 2.415789473684211e-06, |
| "loss": 0.0002, |
| "num_tokens": 1581456.0, |
| "reward": 0.71875, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.71875, |
| "step": 541 |
| }, |
| { |
| "clip_ratio": 0.0008445946150459349, |
| "epoch": 0.07400327689787002, |
| "grad_norm": 0.03024432435631752, |
| "kl": 0.0014385326594492653, |
| "learning_rate": 2.410526315789474e-06, |
| "loss": 0.0, |
| "step": 542 |
| }, |
| { |
| "clip_ratio": 0.00036845837894361466, |
| "epoch": 0.0741398143091207, |
| "grad_norm": 0.030283359810709953, |
| "kl": 0.0013663766912941355, |
| "learning_rate": 2.4052631578947367e-06, |
| "loss": -0.0001, |
| "step": 543 |
| }, |
| { |
| "clip_ratio": 0.00018601190822664648, |
| "epoch": 0.07427635172037138, |
| "grad_norm": 0.03067188523709774, |
| "kl": 0.0013775449333479628, |
| "learning_rate": 2.4000000000000003e-06, |
| "loss": 0.0002, |
| "step": 544 |
| }, |
| { |
| "clip_ratio": 0.0005580357392318547, |
| "completion_length": 119.5625, |
| "epoch": 0.07441288913162207, |
| "grad_norm": 0.02598925493657589, |
| "kl": 0.001481451478866802, |
| "learning_rate": 2.3947368421052635e-06, |
| "loss": 0.0, |
| "num_tokens": 1592442.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 545 |
| }, |
| { |
| "clip_ratio": 0.0001653439103392884, |
| "epoch": 0.07454942654287275, |
| "grad_norm": 0.028425514698028564, |
| "kl": 0.0016474172589369118, |
| "learning_rate": 2.3894736842105266e-06, |
| "loss": 0.0, |
| "step": 546 |
| }, |
| { |
| "clip_ratio": 0.00048688988317735493, |
| "epoch": 0.07468596395412343, |
| "grad_norm": 0.027035797014832497, |
| "kl": 0.0014499265525955707, |
| "learning_rate": 2.38421052631579e-06, |
| "loss": -0.0, |
| "step": 547 |
| }, |
| { |
| "clip_ratio": 0.0005580357392318547, |
| "epoch": 0.07482250136537412, |
| "grad_norm": 0.027835296466946602, |
| "kl": 0.0014104215515544638, |
| "learning_rate": 2.378947368421053e-06, |
| "loss": 0.0, |
| "step": 548 |
| }, |
| { |
| "clip_ratio": 0.001446785288862884, |
| "completion_length": 141.5625, |
| "epoch": 0.0749590387766248, |
| "grad_norm": 0.02882438339293003, |
| "kl": 0.0013727315872529289, |
| "learning_rate": 2.373684210526316e-06, |
| "loss": 0.0001, |
| "num_tokens": 1603960.0, |
| "reward": 0.4625000059604645, |
| "reward_std": 0.23995190858840942, |
| "rewards/reward_fn": 0.4625000059604645, |
| "step": 549 |
| }, |
| { |
| "clip_ratio": 0.0004970216541551054, |
| "epoch": 0.07509557618787548, |
| "grad_norm": 0.028407303616404533, |
| "kl": 0.001324782728261198, |
| "learning_rate": 2.368421052631579e-06, |
| "loss": -0.0, |
| "step": 550 |
| }, |
| { |
| "clip_ratio": 0.0012807868115487508, |
| "epoch": 0.07523211359912615, |
| "grad_norm": 0.028264285996556282, |
| "kl": 0.001328441574514727, |
| "learning_rate": 2.363157894736842e-06, |
| "loss": -0.0001, |
| "step": 551 |
| }, |
| { |
| "clip_ratio": 0.0008249489692389034, |
| "epoch": 0.07536865101037685, |
| "grad_norm": 0.029252443462610245, |
| "kl": 0.0013075096630927874, |
| "learning_rate": 2.357894736842105e-06, |
| "loss": 0.0001, |
| "step": 552 |
| }, |
| { |
| "clip_ratio": 0.0006459349242504686, |
| "completion_length": 131.78125, |
| "epoch": 0.07550518842162753, |
| "grad_norm": 0.030669942498207092, |
| "kl": 0.0012164198715254315, |
| "learning_rate": 2.352631578947369e-06, |
| "loss": 0.0, |
| "num_tokens": 1615629.0, |
| "reward": 0.71875, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.71875, |
| "step": 553 |
| }, |
| { |
| "clip_ratio": 0.0011245697824051604, |
| "epoch": 0.0756417258328782, |
| "grad_norm": 0.030605750158429146, |
| "kl": 0.00125411778481066, |
| "learning_rate": 2.347368421052632e-06, |
| "loss": -0.0001, |
| "step": 554 |
| }, |
| { |
| "clip_ratio": 0.00022163119865581393, |
| "epoch": 0.0757782632441289, |
| "grad_norm": 0.030652441084384918, |
| "kl": 0.0013444009427985293, |
| "learning_rate": 2.342105263157895e-06, |
| "loss": -0.0001, |
| "step": 555 |
| }, |
| { |
| "clip_ratio": 0.0008194020629161969, |
| "epoch": 0.07591480065537957, |
| "grad_norm": 0.03041120246052742, |
| "kl": 0.0011914784172404325, |
| "learning_rate": 2.3368421052631583e-06, |
| "loss": -0.0002, |
| "step": 556 |
| }, |
| { |
| "clip_ratio": 0.00019409938249737024, |
| "completion_length": 153.25, |
| "epoch": 0.07605133806663025, |
| "grad_norm": 0.01942029595375061, |
| "kl": 0.0013100661726639373, |
| "learning_rate": 2.331578947368421e-06, |
| "loss": 0.0002, |
| "num_tokens": 1628237.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 557 |
| }, |
| { |
| "clip_ratio": 0.00019409938249737024, |
| "epoch": 0.07618787547788095, |
| "grad_norm": 0.019458364695310593, |
| "kl": 0.0011170834286531317, |
| "learning_rate": 2.326315789473684e-06, |
| "loss": 0.0002, |
| "step": 558 |
| }, |
| { |
| "clip_ratio": 0.0006215237954165787, |
| "epoch": 0.07632441288913162, |
| "grad_norm": 0.020504020154476166, |
| "kl": 0.0012800872691514087, |
| "learning_rate": 2.3210526315789473e-06, |
| "loss": 0.0002, |
| "step": 559 |
| }, |
| { |
| "clip_ratio": 0.00019409938249737024, |
| "epoch": 0.0764609503003823, |
| "grad_norm": 0.019755452871322632, |
| "kl": 0.001140367172411061, |
| "learning_rate": 2.3157894736842105e-06, |
| "loss": 0.0002, |
| "step": 560 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 134.96875, |
| "epoch": 0.07659748771163298, |
| "grad_norm": 0.02310808189213276, |
| "kl": 0.0012227943288962706, |
| "learning_rate": 2.310526315789474e-06, |
| "loss": -0.0001, |
| "num_tokens": 1639516.0, |
| "reward": 0.887499988079071, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.887499988079071, |
| "step": 561 |
| }, |
| { |
| "clip_ratio": 0.0005299604235915467, |
| "epoch": 0.07673402512288367, |
| "grad_norm": 0.022301802411675453, |
| "kl": 0.0012091731859982247, |
| "learning_rate": 2.3052631578947373e-06, |
| "loss": -0.0002, |
| "step": 562 |
| }, |
| { |
| "clip_ratio": 0.00021701389050576836, |
| "epoch": 0.07687056253413435, |
| "grad_norm": 0.02227598801255226, |
| "kl": 0.0011962663684244035, |
| "learning_rate": 2.3000000000000004e-06, |
| "loss": -0.0001, |
| "step": 563 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07700709994538503, |
| "grad_norm": 0.02261110581457615, |
| "kl": 0.0012483229429562925, |
| "learning_rate": 2.294736842105263e-06, |
| "loss": -0.0002, |
| "step": 564 |
| }, |
| { |
| "clip_ratio": 0.00038580247201025486, |
| "completion_length": 111.53125, |
| "epoch": 0.07714363735663572, |
| "grad_norm": 0.03374376520514488, |
| "kl": 0.0012109982199035585, |
| "learning_rate": 2.2894736842105263e-06, |
| "loss": 0.0, |
| "num_tokens": 1649869.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 565 |
| }, |
| { |
| "clip_ratio": 0.0007292090740520507, |
| "epoch": 0.0772801747678864, |
| "grad_norm": 0.03348137438297272, |
| "kl": 0.001319482877079281, |
| "learning_rate": 2.2842105263157895e-06, |
| "loss": 0.0001, |
| "step": 566 |
| }, |
| { |
| "clip_ratio": 0.000866827875142917, |
| "epoch": 0.07741671217913708, |
| "grad_norm": 0.032686445862054825, |
| "kl": 0.0012758291504724184, |
| "learning_rate": 2.2789473684210527e-06, |
| "loss": 0.0001, |
| "step": 567 |
| }, |
| { |
| "clip_ratio": 0.000631865463219583, |
| "epoch": 0.07755324959038777, |
| "grad_norm": 0.03336594998836517, |
| "kl": 0.0013503911413863534, |
| "learning_rate": 2.273684210526316e-06, |
| "loss": 0.0001, |
| "step": 568 |
| }, |
| { |
| "clip_ratio": 0.0005172694654902443, |
| "completion_length": 138.75, |
| "epoch": 0.07768978700163845, |
| "grad_norm": 0.0384264700114727, |
| "kl": 0.0013226108276285231, |
| "learning_rate": 2.268421052631579e-06, |
| "loss": -0.0001, |
| "num_tokens": 1661965.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 569 |
| }, |
| { |
| "clip_ratio": 0.0009963809716282412, |
| "epoch": 0.07782632441288913, |
| "grad_norm": 0.03718084841966629, |
| "kl": 0.0013709642798858113, |
| "learning_rate": 2.2631578947368426e-06, |
| "loss": 0.0001, |
| "step": 570 |
| }, |
| { |
| "clip_ratio": 0.0006437876872951165, |
| "epoch": 0.07796286182413982, |
| "grad_norm": 0.03695379197597504, |
| "kl": 0.0014017439098097384, |
| "learning_rate": 2.2578947368421057e-06, |
| "loss": 0.0002, |
| "step": 571 |
| }, |
| { |
| "clip_ratio": 0.0006249233265407383, |
| "epoch": 0.0780993992353905, |
| "grad_norm": 0.03681650757789612, |
| "kl": 0.0013024672862229636, |
| "learning_rate": 2.2526315789473685e-06, |
| "loss": -0.0001, |
| "step": 572 |
| }, |
| { |
| "clip_ratio": 0.0007458344407496043, |
| "completion_length": 131.125, |
| "epoch": 0.07823593664664118, |
| "grad_norm": 0.033540453761816025, |
| "kl": 0.0015056260308483616, |
| "learning_rate": 2.2473684210526316e-06, |
| "loss": 0.0001, |
| "num_tokens": 1673245.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 573 |
| }, |
| { |
| "clip_ratio": 0.0011247631337027997, |
| "epoch": 0.07837247405789186, |
| "grad_norm": 0.034303613007068634, |
| "kl": 0.0013787490188406082, |
| "learning_rate": 2.242105263157895e-06, |
| "loss": 0.0001, |
| "step": 574 |
| }, |
| { |
| "clip_ratio": 0.0005023798657930456, |
| "epoch": 0.07850901146914255, |
| "grad_norm": 0.03354345262050629, |
| "kl": 0.0014687197344755987, |
| "learning_rate": 2.236842105263158e-06, |
| "loss": 0.0002, |
| "step": 575 |
| }, |
| { |
| "clip_ratio": 0.0006665511391474865, |
| "epoch": 0.07864554888039323, |
| "grad_norm": 0.034005530178546906, |
| "kl": 0.0014215231167327147, |
| "learning_rate": 2.231578947368421e-06, |
| "loss": -0.0, |
| "step": 576 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 119.21875, |
| "epoch": 0.0787820862916439, |
| "grad_norm": 0.028434976935386658, |
| "kl": 0.0013502443598554237, |
| "learning_rate": 2.2263157894736843e-06, |
| "loss": 0.0001, |
| "num_tokens": 1684708.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 577 |
| }, |
| { |
| "clip_ratio": 0.0005482456181198359, |
| "epoch": 0.0789186237028946, |
| "grad_norm": 0.028591668233275414, |
| "kl": 0.0012434296741048456, |
| "learning_rate": 2.221052631578948e-06, |
| "loss": -0.0, |
| "step": 578 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07905516111414528, |
| "grad_norm": 0.028464889153838158, |
| "kl": 0.0013504233384082909, |
| "learning_rate": 2.2157894736842106e-06, |
| "loss": -0.0002, |
| "step": 579 |
| }, |
| { |
| "clip_ratio": 0.0005482456181198359, |
| "epoch": 0.07919169852539595, |
| "grad_norm": 0.0283973291516304, |
| "kl": 0.0013455405560307554, |
| "learning_rate": 2.2105263157894738e-06, |
| "loss": -0.0001, |
| "step": 580 |
| }, |
| { |
| "clip_ratio": 0.0003379267000127584, |
| "completion_length": 144.90625, |
| "epoch": 0.07932823593664665, |
| "grad_norm": 0.01949334517121315, |
| "kl": 0.0012513340170698939, |
| "learning_rate": 2.205263157894737e-06, |
| "loss": 0.0001, |
| "num_tokens": 1697297.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 581 |
| }, |
| { |
| "clip_ratio": 0.0006014183891238645, |
| "epoch": 0.07946477334789732, |
| "grad_norm": 0.01936924457550049, |
| "kl": 0.0012721861930913292, |
| "learning_rate": 2.2e-06, |
| "loss": 0.0, |
| "step": 582 |
| }, |
| { |
| "clip_ratio": 0.0005715087900171056, |
| "epoch": 0.079601310759148, |
| "grad_norm": 0.018492933362722397, |
| "kl": 0.0012525502661446808, |
| "learning_rate": 2.1947368421052633e-06, |
| "loss": 0.0001, |
| "step": 583 |
| }, |
| { |
| "clip_ratio": 0.00016622339899186045, |
| "epoch": 0.0797378481703987, |
| "grad_norm": 0.017930587753653526, |
| "kl": 0.0011951930664508836, |
| "learning_rate": 2.1894736842105264e-06, |
| "loss": -0.0, |
| "step": 584 |
| }, |
| { |
| "clip_ratio": 0.0004695706011261791, |
| "completion_length": 139.1875, |
| "epoch": 0.07987438558164937, |
| "grad_norm": 0.028043018653988838, |
| "kl": 0.0011986813478870317, |
| "learning_rate": 2.1842105263157896e-06, |
| "loss": 0.0, |
| "num_tokens": 1708779.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 585 |
| }, |
| { |
| "clip_ratio": 0.000575878111703787, |
| "epoch": 0.08001092299290005, |
| "grad_norm": 0.027894066646695137, |
| "kl": 0.0012194329829071648, |
| "learning_rate": 2.1789473684210528e-06, |
| "loss": 0.0002, |
| "step": 586 |
| }, |
| { |
| "clip_ratio": 0.00023320894979406148, |
| "epoch": 0.08014746040415073, |
| "grad_norm": 0.02750025875866413, |
| "kl": 0.0012120107594455476, |
| "learning_rate": 2.173684210526316e-06, |
| "loss": -0.0001, |
| "step": 587 |
| }, |
| { |
| "clip_ratio": 0.0008329564152518287, |
| "epoch": 0.08028399781540142, |
| "grad_norm": 0.028404943645000458, |
| "kl": 0.0013067316140222829, |
| "learning_rate": 2.168421052631579e-06, |
| "loss": 0.0, |
| "step": 588 |
| }, |
| { |
| "clip_ratio": 0.0001849112450145185, |
| "completion_length": 112.21875, |
| "epoch": 0.0804205352266521, |
| "grad_norm": 0.025281289592385292, |
| "kl": 0.0012155929061918869, |
| "learning_rate": 2.1631578947368423e-06, |
| "loss": -0.0001, |
| "num_tokens": 1720098.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 589 |
| }, |
| { |
| "clip_ratio": 0.0004163927223999053, |
| "epoch": 0.08055707263790278, |
| "grad_norm": 0.025050563737750053, |
| "kl": 0.0011125751880172174, |
| "learning_rate": 2.1578947368421054e-06, |
| "loss": -0.0002, |
| "step": 590 |
| }, |
| { |
| "clip_ratio": 0.0004163927223999053, |
| "epoch": 0.08069361004915347, |
| "grad_norm": 0.025595294311642647, |
| "kl": 0.001212971978020505, |
| "learning_rate": 2.1526315789473686e-06, |
| "loss": -0.0001, |
| "step": 591 |
| }, |
| { |
| "clip_ratio": 0.0007168735028244555, |
| "epoch": 0.08083014746040415, |
| "grad_norm": 0.024896765127778053, |
| "kl": 0.0012561739672491967, |
| "learning_rate": 2.1473684210526317e-06, |
| "loss": -0.0001, |
| "step": 592 |
| }, |
| { |
| "clip_ratio": 0.0005804953689221293, |
| "completion_length": 133.1875, |
| "epoch": 0.08096668487165483, |
| "grad_norm": 0.03213340416550636, |
| "kl": 0.0012622286103578517, |
| "learning_rate": 2.142105263157895e-06, |
| "loss": -0.0001, |
| "num_tokens": 1731732.0, |
| "reward": 0.6031249761581421, |
| "reward_std": 0.2962018847465515, |
| "rewards/reward_fn": 0.6031249761581421, |
| "step": 593 |
| }, |
| { |
| "clip_ratio": 0.00047927717969287187, |
| "epoch": 0.08110322228290552, |
| "grad_norm": 0.031801752746105194, |
| "kl": 0.0011793114590545883, |
| "learning_rate": 2.136842105263158e-06, |
| "loss": -0.0001, |
| "step": 594 |
| }, |
| { |
| "clip_ratio": 0.0005405196279753, |
| "epoch": 0.0812397596941562, |
| "grad_norm": 0.03253817930817604, |
| "kl": 0.0011914531751244795, |
| "learning_rate": 2.1315789473684212e-06, |
| "loss": -0.0001, |
| "step": 595 |
| }, |
| { |
| "clip_ratio": 0.0007670347695238888, |
| "epoch": 0.08137629710540688, |
| "grad_norm": 0.03316066786646843, |
| "kl": 0.0012645062852243427, |
| "learning_rate": 2.1263157894736844e-06, |
| "loss": -0.0, |
| "step": 596 |
| }, |
| { |
| "clip_ratio": 0.0003789131442317739, |
| "completion_length": 144.28125, |
| "epoch": 0.08151283451665757, |
| "grad_norm": 0.028041541576385498, |
| "kl": 0.001233616606441501, |
| "learning_rate": 2.1210526315789476e-06, |
| "loss": -0.0, |
| "num_tokens": 1744297.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 597 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08164937192790825, |
| "grad_norm": 0.028079692274332047, |
| "kl": 0.0011803826682807994, |
| "learning_rate": 2.1157894736842107e-06, |
| "loss": 0.0, |
| "step": 598 |
| }, |
| { |
| "clip_ratio": 0.0004720039287349209, |
| "epoch": 0.08178590933915893, |
| "grad_norm": 0.02811983786523342, |
| "kl": 0.0012397360787872458, |
| "learning_rate": 2.110526315789474e-06, |
| "loss": -0.0002, |
| "step": 599 |
| }, |
| { |
| "clip_ratio": 0.0004097465134691447, |
| "epoch": 0.0819224467504096, |
| "grad_norm": 0.028073711320757866, |
| "kl": 0.0012096571817892254, |
| "learning_rate": 2.105263157894737e-06, |
| "loss": -0.0, |
| "step": 600 |
| }, |
| { |
| "clip_ratio": 0.0010307422780897468, |
| "completion_length": 129.46875, |
| "epoch": 0.0820589841616603, |
| "grad_norm": 0.03012477047741413, |
| "kl": 0.001331004872554331, |
| "learning_rate": 2.1000000000000002e-06, |
| "loss": 0.0002, |
| "num_tokens": 1755712.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 601 |
| }, |
| { |
| "clip_ratio": 0.0010868488752748817, |
| "epoch": 0.08219552157291098, |
| "grad_norm": 0.027972938492894173, |
| "kl": 0.0012423287271303707, |
| "learning_rate": 2.0947368421052634e-06, |
| "loss": 0.0001, |
| "step": 602 |
| }, |
| { |
| "clip_ratio": 0.0017305946748820134, |
| "epoch": 0.08233205898416165, |
| "grad_norm": 0.027943221852183342, |
| "kl": 0.0013830821371811908, |
| "learning_rate": 2.0894736842105266e-06, |
| "loss": 0.0002, |
| "step": 603 |
| }, |
| { |
| "clip_ratio": 0.0005490178009495139, |
| "epoch": 0.08246859639541235, |
| "grad_norm": 0.028850063681602478, |
| "kl": 0.0013167489851184655, |
| "learning_rate": 2.0842105263157897e-06, |
| "loss": -0.0, |
| "step": 604 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 120.90625, |
| "epoch": 0.08260513380666303, |
| "grad_norm": 0.024748941883444786, |
| "kl": 0.0014295073833636707, |
| "learning_rate": 2.078947368421053e-06, |
| "loss": 0.0, |
| "num_tokens": 1766085.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 605 |
| }, |
| { |
| "clip_ratio": 0.00019654087373055518, |
| "epoch": 0.0827416712179137, |
| "grad_norm": 0.02504793554544449, |
| "kl": 0.001298841527386685, |
| "learning_rate": 2.073684210526316e-06, |
| "loss": -0.0, |
| "step": 606 |
| }, |
| { |
| "clip_ratio": 0.00020559210679493845, |
| "epoch": 0.0828782086291644, |
| "grad_norm": 0.02473580837249756, |
| "kl": 0.0013759256125922548, |
| "learning_rate": 2.068421052631579e-06, |
| "loss": 0.0, |
| "step": 607 |
| }, |
| { |
| "clip_ratio": 0.00020559210679493845, |
| "epoch": 0.08301474604041507, |
| "grad_norm": 0.024983009323477745, |
| "kl": 0.0013146149885869818, |
| "learning_rate": 2.0631578947368424e-06, |
| "loss": -0.0001, |
| "step": 608 |
| }, |
| { |
| "clip_ratio": 0.00013950893480796367, |
| "completion_length": 159.40625, |
| "epoch": 0.08315128345166575, |
| "grad_norm": 0.02585506998002529, |
| "kl": 0.0011791864890255965, |
| "learning_rate": 2.0578947368421055e-06, |
| "loss": -0.0002, |
| "num_tokens": 1778654.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 609 |
| }, |
| { |
| "clip_ratio": 0.000621384650003165, |
| "epoch": 0.08328782086291645, |
| "grad_norm": 0.026020504534244537, |
| "kl": 0.0011703061281878036, |
| "learning_rate": 2.0526315789473687e-06, |
| "loss": -0.0001, |
| "step": 610 |
| }, |
| { |
| "clip_ratio": 0.0009574061550665647, |
| "epoch": 0.08342435827416712, |
| "grad_norm": 0.026134071871638298, |
| "kl": 0.0012730637799904798, |
| "learning_rate": 2.047368421052632e-06, |
| "loss": -0.0001, |
| "step": 611 |
| }, |
| { |
| "clip_ratio": 0.00047553043987136334, |
| "epoch": 0.0835608956854178, |
| "grad_norm": 0.02622632496058941, |
| "kl": 0.0011777592935686698, |
| "learning_rate": 2.042105263157895e-06, |
| "loss": -0.0001, |
| "step": 612 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 133.28125, |
| "epoch": 0.08369743309666848, |
| "grad_norm": 0.029103118926286697, |
| "kl": 0.0013813998102705227, |
| "learning_rate": 2.036842105263158e-06, |
| "loss": -0.0, |
| "num_tokens": 1789519.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.2511056959629059, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 613 |
| }, |
| { |
| "clip_ratio": 0.0003215289398212917, |
| "epoch": 0.08383397050791917, |
| "grad_norm": 0.028894973918795586, |
| "kl": 0.0013992627809784608, |
| "learning_rate": 2.031578947368421e-06, |
| "loss": -0.0001, |
| "step": 614 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08397050791916985, |
| "grad_norm": 0.02881738543510437, |
| "kl": 0.0014436542860494228, |
| "learning_rate": 2.026315789473684e-06, |
| "loss": -0.0003, |
| "step": 615 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08410704533042053, |
| "grad_norm": 0.029226887971162796, |
| "kl": 0.0015083678954397328, |
| "learning_rate": 2.0210526315789477e-06, |
| "loss": -0.0002, |
| "step": 616 |
| }, |
| { |
| "clip_ratio": 0.0003229463181924075, |
| "completion_length": 168.1875, |
| "epoch": 0.08424358274167122, |
| "grad_norm": 0.024407584220170975, |
| "kl": 0.0014167944955261191, |
| "learning_rate": 2.015789473684211e-06, |
| "loss": -0.0, |
| "num_tokens": 1802857.0, |
| "reward": 0.4625000059604645, |
| "reward_std": 0.17499999701976776, |
| "rewards/reward_fn": 0.4625000059604645, |
| "step": 617 |
| }, |
| { |
| "clip_ratio": 0.00030754510225960985, |
| "epoch": 0.0843801201529219, |
| "grad_norm": 0.025380659848451614, |
| "kl": 0.0012805712885892717, |
| "learning_rate": 2.010526315789474e-06, |
| "loss": 0.0001, |
| "step": 618 |
| }, |
| { |
| "clip_ratio": 0.00031566374673275277, |
| "epoch": 0.08451665756417258, |
| "grad_norm": 0.025637686252593994, |
| "kl": 0.0012550040901260218, |
| "learning_rate": 2.005263157894737e-06, |
| "loss": 0.0001, |
| "step": 619 |
| }, |
| { |
| "clip_ratio": 0.00017959770048037171, |
| "epoch": 0.08465319497542327, |
| "grad_norm": 0.02544740028679371, |
| "kl": 0.0013416227866400732, |
| "learning_rate": 2.0000000000000003e-06, |
| "loss": -0.0001, |
| "step": 620 |
| }, |
| { |
| "clip_ratio": 0.000400834578613285, |
| "completion_length": 143.625, |
| "epoch": 0.08478973238667395, |
| "grad_norm": 0.025843657553195953, |
| "kl": 0.001131466372498835, |
| "learning_rate": 1.994736842105263e-06, |
| "loss": 0.0, |
| "num_tokens": 1815005.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 621 |
| }, |
| { |
| "clip_ratio": 0.0006174136869958602, |
| "epoch": 0.08492626979792463, |
| "grad_norm": 0.02638258785009384, |
| "kl": 0.0010728317356551997, |
| "learning_rate": 1.9894736842105262e-06, |
| "loss": 0.0, |
| "step": 622 |
| }, |
| { |
| "clip_ratio": 0.0002513130530132912, |
| "epoch": 0.08506280720917532, |
| "grad_norm": 0.025244932621717453, |
| "kl": 0.0011344658514644834, |
| "learning_rate": 1.9842105263157894e-06, |
| "loss": -0.0001, |
| "step": 623 |
| }, |
| { |
| "clip_ratio": 0.00045424259587889537, |
| "epoch": 0.085199344620426, |
| "grad_norm": 0.025821218267083168, |
| "kl": 0.0011071752214775188, |
| "learning_rate": 1.978947368421053e-06, |
| "loss": 0.0, |
| "step": 624 |
| }, |
| { |
| "clip_ratio": 0.00040070006798487157, |
| "completion_length": 138.96875, |
| "epoch": 0.08533588203167668, |
| "grad_norm": 0.020593425258994102, |
| "kl": 0.0014952101555536501, |
| "learning_rate": 1.973684210526316e-06, |
| "loss": 0.0001, |
| "num_tokens": 1827092.0, |
| "reward": 0.6906249523162842, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.6906249523162842, |
| "step": 625 |
| }, |
| { |
| "clip_ratio": 0.0007846917069400661, |
| "epoch": 0.08547241944292736, |
| "grad_norm": 0.01970861665904522, |
| "kl": 0.0015636316857126076, |
| "learning_rate": 1.9684210526315793e-06, |
| "loss": 0.0002, |
| "step": 626 |
| }, |
| { |
| "clip_ratio": 0.00026483050896786153, |
| "epoch": 0.08560895685417805, |
| "grad_norm": 0.020389128476381302, |
| "kl": 0.0015640047313354444, |
| "learning_rate": 1.9631578947368425e-06, |
| "loss": 0.0001, |
| "step": 627 |
| }, |
| { |
| "clip_ratio": 0.0005353983433451504, |
| "epoch": 0.08574549426542873, |
| "grad_norm": 0.020528005436062813, |
| "kl": 0.001569879857925116, |
| "learning_rate": 1.9578947368421052e-06, |
| "loss": 0.0003, |
| "step": 628 |
| }, |
| { |
| "clip_ratio": 0.00027173911803402007, |
| "completion_length": 106.28125, |
| "epoch": 0.0858820316766794, |
| "grad_norm": 0.025182850658893585, |
| "kl": 0.0017427417806175072, |
| "learning_rate": 1.9526315789473684e-06, |
| "loss": 0.0001, |
| "num_tokens": 1838041.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 629 |
| }, |
| { |
| "clip_ratio": 0.00015394088404718786, |
| "epoch": 0.0860185690879301, |
| "grad_norm": 0.025246020406484604, |
| "kl": 0.0017945856025107787, |
| "learning_rate": 1.9473684210526315e-06, |
| "loss": -0.0, |
| "step": 630 |
| }, |
| { |
| "clip_ratio": 0.00027173911803402007, |
| "epoch": 0.08615510649918078, |
| "grad_norm": 0.02506876550614834, |
| "kl": 0.0017630000711506, |
| "learning_rate": 1.9421052631578947e-06, |
| "loss": 0.0, |
| "step": 631 |
| }, |
| { |
| "clip_ratio": 0.00028153153834864497, |
| "epoch": 0.08629164391043145, |
| "grad_norm": 0.02513444982469082, |
| "kl": 0.0018713257522904314, |
| "learning_rate": 1.936842105263158e-06, |
| "loss": -0.0, |
| "step": 632 |
| }, |
| { |
| "clip_ratio": 0.0001806358341127634, |
| "completion_length": 133.9375, |
| "epoch": 0.08642818132168215, |
| "grad_norm": 0.030734168365597725, |
| "kl": 0.0014602373357774923, |
| "learning_rate": 1.9315789473684215e-06, |
| "loss": -0.0, |
| "num_tokens": 1849627.0, |
| "reward": 0.71875, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.71875, |
| "step": 633 |
| }, |
| { |
| "clip_ratio": 0.0008967998146545142, |
| "epoch": 0.08656471873293282, |
| "grad_norm": 0.031097473576664925, |
| "kl": 0.001530306273707538, |
| "learning_rate": 1.9263157894736846e-06, |
| "loss": -0.0001, |
| "step": 634 |
| }, |
| { |
| "clip_ratio": 0.0002281021879753098, |
| "epoch": 0.0867012561441835, |
| "grad_norm": 0.031329941004514694, |
| "kl": 0.0013291898703755578, |
| "learning_rate": 1.9210526315789474e-06, |
| "loss": -0.0001, |
| "step": 635 |
| }, |
| { |
| "clip_ratio": 0.0004247764591127634, |
| "epoch": 0.0868377935554342, |
| "grad_norm": 0.030992189422249794, |
| "kl": 0.0014359870256157592, |
| "learning_rate": 1.9157894736842105e-06, |
| "loss": -0.0004, |
| "step": 636 |
| }, |
| { |
| "clip_ratio": 0.0003124999930150807, |
| "completion_length": 129.71875, |
| "epoch": 0.08697433096668487, |
| "grad_norm": 0.02779513970017433, |
| "kl": 0.0014640076806244906, |
| "learning_rate": 1.9105263157894737e-06, |
| "loss": 0.0001, |
| "num_tokens": 1861258.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 637 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08711086837793555, |
| "grad_norm": 0.02775580994784832, |
| "kl": 0.001482383013353683, |
| "learning_rate": 1.905263157894737e-06, |
| "loss": 0.0001, |
| "step": 638 |
| }, |
| { |
| "clip_ratio": 0.0001453488366678357, |
| "epoch": 0.08724740578918623, |
| "grad_norm": 0.027669286355376244, |
| "kl": 0.0014783034021093044, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": 0.0001, |
| "step": 639 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08738394320043692, |
| "grad_norm": 0.02769199199974537, |
| "kl": 0.0014845102041363134, |
| "learning_rate": 1.8947368421052634e-06, |
| "loss": 0.0001, |
| "step": 640 |
| }, |
| { |
| "clip_ratio": 0.0009185846138279885, |
| "completion_length": 128.46875, |
| "epoch": 0.0875204806116876, |
| "grad_norm": 0.033943865448236465, |
| "kl": 0.0014185677864588797, |
| "learning_rate": 1.8894736842105266e-06, |
| "loss": 0.0, |
| "num_tokens": 1872497.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 641 |
| }, |
| { |
| "clip_ratio": 0.0007410277903545648, |
| "epoch": 0.08765701802293828, |
| "grad_norm": 0.033117737621068954, |
| "kl": 0.0015516698476858437, |
| "learning_rate": 1.8842105263157895e-06, |
| "loss": -0.0, |
| "step": 642 |
| }, |
| { |
| "clip_ratio": 0.0007410277903545648, |
| "epoch": 0.08779355543418897, |
| "grad_norm": 0.032690513879060745, |
| "kl": 0.0014765476635147934, |
| "learning_rate": 1.8789473684210527e-06, |
| "loss": -0.0002, |
| "step": 643 |
| }, |
| { |
| "clip_ratio": 0.0007410277903545648, |
| "epoch": 0.08793009284543965, |
| "grad_norm": 0.033511463552713394, |
| "kl": 0.0015954046502884012, |
| "learning_rate": 1.8736842105263158e-06, |
| "loss": -0.0002, |
| "step": 644 |
| }, |
| { |
| "clip_ratio": 0.0007349282532231882, |
| "completion_length": 117.1875, |
| "epoch": 0.08806663025669033, |
| "grad_norm": 0.04200991615653038, |
| "kl": 0.0018133009270968614, |
| "learning_rate": 1.868421052631579e-06, |
| "loss": 0.0002, |
| "num_tokens": 1883011.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.2337018847465515, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 645 |
| }, |
| { |
| "clip_ratio": 0.0011147170735057443, |
| "epoch": 0.08820316766794102, |
| "grad_norm": 0.03769839555025101, |
| "kl": 0.0018559084401204018, |
| "learning_rate": 1.8631578947368424e-06, |
| "loss": -0.0001, |
| "step": 646 |
| }, |
| { |
| "clip_ratio": 0.0013395372225204483, |
| "epoch": 0.0883397050791917, |
| "grad_norm": 0.03914198279380798, |
| "kl": 0.0018133489347746945, |
| "learning_rate": 1.8578947368421055e-06, |
| "loss": 0.0001, |
| "step": 647 |
| }, |
| { |
| "clip_ratio": 0.001044334188918583, |
| "epoch": 0.08847624249044238, |
| "grad_norm": 0.03948010131716728, |
| "kl": 0.0017424372199457139, |
| "learning_rate": 1.8526315789473687e-06, |
| "loss": 0.0001, |
| "step": 648 |
| }, |
| { |
| "clip_ratio": 0.0009479038853896782, |
| "completion_length": 147.8125, |
| "epoch": 0.08861277990169307, |
| "grad_norm": 0.034093838185071945, |
| "kl": 0.0017491482576588169, |
| "learning_rate": 1.8473684210526319e-06, |
| "loss": 0.0002, |
| "num_tokens": 1895205.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 649 |
| }, |
| { |
| "clip_ratio": 0.0013839395687682554, |
| "epoch": 0.08874931731294375, |
| "grad_norm": 0.034466780722141266, |
| "kl": 0.0017224287621502299, |
| "learning_rate": 1.8421052631578948e-06, |
| "loss": 0.0003, |
| "step": 650 |
| }, |
| { |
| "clip_ratio": 0.0007027992105577141, |
| "epoch": 0.08888585472419443, |
| "grad_norm": 0.034083131700754166, |
| "kl": 0.0017858712380984798, |
| "learning_rate": 1.836842105263158e-06, |
| "loss": 0.0001, |
| "step": 651 |
| }, |
| { |
| "clip_ratio": 0.00064598981407471, |
| "epoch": 0.0890223921354451, |
| "grad_norm": 0.03420592471957207, |
| "kl": 0.0017675116159807658, |
| "learning_rate": 1.8315789473684211e-06, |
| "loss": 0.0, |
| "step": 652 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.78125, |
| "epoch": 0.0891589295466958, |
| "grad_norm": 0.025346998125314713, |
| "kl": 0.001804993613404804, |
| "learning_rate": 1.8263157894736843e-06, |
| "loss": -0.0001, |
| "num_tokens": 1906734.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 653 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08929546695794648, |
| "grad_norm": 0.025825245305895805, |
| "kl": 0.001727437149384059, |
| "learning_rate": 1.8210526315789475e-06, |
| "loss": -0.0001, |
| "step": 654 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08943200436919715, |
| "grad_norm": 0.025717921555042267, |
| "kl": 0.0016938845965341898, |
| "learning_rate": 1.8157894736842109e-06, |
| "loss": 0.0, |
| "step": 655 |
| }, |
| { |
| "clip_ratio": 0.00037425151094794273, |
| "epoch": 0.08956854178044785, |
| "grad_norm": 0.02527354285120964, |
| "kl": 0.0016179548201762373, |
| "learning_rate": 1.810526315789474e-06, |
| "loss": -0.0001, |
| "step": 656 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 133.1875, |
| "epoch": 0.08970507919169853, |
| "grad_norm": 0.03043227270245552, |
| "kl": 0.0015594559863529867, |
| "learning_rate": 1.805263157894737e-06, |
| "loss": -0.0, |
| "num_tokens": 1918476.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 657 |
| }, |
| { |
| "clip_ratio": 0.0001806358341127634, |
| "epoch": 0.0898416166029492, |
| "grad_norm": 0.03057057410478592, |
| "kl": 0.0016266269267362077, |
| "learning_rate": 1.8000000000000001e-06, |
| "loss": -0.0001, |
| "step": 658 |
| }, |
| { |
| "clip_ratio": 0.0009754410712048411, |
| "epoch": 0.0899781540141999, |
| "grad_norm": 0.031286321580410004, |
| "kl": 0.0016341756581823574, |
| "learning_rate": 1.7947368421052633e-06, |
| "loss": -0.0, |
| "step": 659 |
| }, |
| { |
| "clip_ratio": 0.0006547989178216085, |
| "epoch": 0.09011469142545057, |
| "grad_norm": 0.030410924926400185, |
| "kl": 0.0016616588927718112, |
| "learning_rate": 1.7894736842105265e-06, |
| "loss": -0.0, |
| "step": 660 |
| }, |
| { |
| "clip_ratio": 0.0005230949609540403, |
| "completion_length": 138.0, |
| "epoch": 0.09025122883670125, |
| "grad_norm": 0.034563641995191574, |
| "kl": 0.0014913913328200579, |
| "learning_rate": 1.7842105263157896e-06, |
| "loss": 0.0001, |
| "num_tokens": 1930076.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.298653781414032, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 661 |
| }, |
| { |
| "clip_ratio": 0.000984658210654743, |
| "epoch": 0.09038776624795195, |
| "grad_norm": 0.03449584171175957, |
| "kl": 0.0014851825108053163, |
| "learning_rate": 1.7789473684210528e-06, |
| "loss": 0.0003, |
| "step": 662 |
| }, |
| { |
| "clip_ratio": 0.000619944155914709, |
| "epoch": 0.09052430365920262, |
| "grad_norm": 0.03485969081521034, |
| "kl": 0.0014608390283683548, |
| "learning_rate": 1.7736842105263162e-06, |
| "loss": 0.0, |
| "step": 663 |
| }, |
| { |
| "clip_ratio": 0.0005230949609540403, |
| "epoch": 0.0906608410704533, |
| "grad_norm": 0.03468193858861923, |
| "kl": 0.0013985099394631106, |
| "learning_rate": 1.768421052631579e-06, |
| "loss": -0.0001, |
| "step": 664 |
| }, |
| { |
| "clip_ratio": 0.0016197122313315049, |
| "completion_length": 126.0625, |
| "epoch": 0.09079737848170398, |
| "grad_norm": 0.048120178282260895, |
| "kl": 0.0019150414373143576, |
| "learning_rate": 1.7631578947368423e-06, |
| "loss": 0.0002, |
| "num_tokens": 1941278.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.4372595250606537, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 665 |
| }, |
| { |
| "clip_ratio": 0.0028011579561280087, |
| "epoch": 0.09093391589295467, |
| "grad_norm": 0.04828806221485138, |
| "kl": 0.001905280525534181, |
| "learning_rate": 1.7578947368421054e-06, |
| "loss": 0.0003, |
| "step": 666 |
| }, |
| { |
| "clip_ratio": 0.0024009775515878573, |
| "epoch": 0.09107045330420535, |
| "grad_norm": 0.050769418478012085, |
| "kl": 0.0019341932602401357, |
| "learning_rate": 1.7526315789473686e-06, |
| "loss": 0.0003, |
| "step": 667 |
| }, |
| { |
| "clip_ratio": 0.0010695689998101443, |
| "epoch": 0.09120699071545603, |
| "grad_norm": 0.04936697706580162, |
| "kl": 0.0019743770517379744, |
| "learning_rate": 1.7473684210526318e-06, |
| "loss": -0.0001, |
| "step": 668 |
| }, |
| { |
| "clip_ratio": 0.0007503781234845519, |
| "completion_length": 140.65625, |
| "epoch": 0.09134352812670672, |
| "grad_norm": 0.03651706874370575, |
| "kl": 0.002162061420676764, |
| "learning_rate": 1.742105263157895e-06, |
| "loss": 0.0003, |
| "num_tokens": 1953231.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.25110572576522827, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 669 |
| }, |
| { |
| "clip_ratio": 0.000699896685546264, |
| "epoch": 0.0914800655379574, |
| "grad_norm": 0.03636300936341286, |
| "kl": 0.0022203527332749218, |
| "learning_rate": 1.736842105263158e-06, |
| "loss": 0.0002, |
| "step": 670 |
| }, |
| { |
| "clip_ratio": 0.00034614611649885774, |
| "epoch": 0.09161660294920808, |
| "grad_norm": 0.037476662546396255, |
| "kl": 0.002253743024994037, |
| "learning_rate": 1.731578947368421e-06, |
| "loss": 0.0003, |
| "step": 671 |
| }, |
| { |
| "clip_ratio": 0.0007263705774676055, |
| "epoch": 0.09175314036045877, |
| "grad_norm": 0.03752395883202553, |
| "kl": 0.0022044695178919937, |
| "learning_rate": 1.7263157894736842e-06, |
| "loss": 0.0002, |
| "step": 672 |
| }, |
| { |
| "clip_ratio": 0.0023742812336422503, |
| "completion_length": 128.5, |
| "epoch": 0.09188967777170945, |
| "grad_norm": 0.04179362207651138, |
| "kl": 0.0015345551819336833, |
| "learning_rate": 1.7210526315789474e-06, |
| "loss": 0.0001, |
| "num_tokens": 1964579.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.35490381717681885, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 673 |
| }, |
| { |
| "clip_ratio": 0.0012075189006282017, |
| "epoch": 0.09202621518296013, |
| "grad_norm": 0.04128982126712799, |
| "kl": 0.0015574862100038445, |
| "learning_rate": 1.7157894736842107e-06, |
| "loss": 0.0, |
| "step": 674 |
| }, |
| { |
| "clip_ratio": 0.0009616574534447864, |
| "epoch": 0.09216275259421082, |
| "grad_norm": 0.04147768020629883, |
| "kl": 0.0015054902996780584, |
| "learning_rate": 1.710526315789474e-06, |
| "loss": -0.0, |
| "step": 675 |
| }, |
| { |
| "clip_ratio": 0.0014374384481925517, |
| "epoch": 0.0922992900054615, |
| "grad_norm": 0.041720401495695114, |
| "kl": 0.0014951720877434127, |
| "learning_rate": 1.705263157894737e-06, |
| "loss": 0.0001, |
| "step": 676 |
| }, |
| { |
| "clip_ratio": 0.0004928899579681456, |
| "completion_length": 128.59375, |
| "epoch": 0.09243582741671218, |
| "grad_norm": 0.02729075774550438, |
| "kl": 0.0019795900498138508, |
| "learning_rate": 1.7000000000000002e-06, |
| "loss": 0.0, |
| "num_tokens": 1975790.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 677 |
| }, |
| { |
| "clip_ratio": 0.0010389106319053099, |
| "epoch": 0.09257236482796286, |
| "grad_norm": 0.027434304356575012, |
| "kl": 0.001802716407837579, |
| "learning_rate": 1.6947368421052632e-06, |
| "loss": 0.0003, |
| "step": 678 |
| }, |
| { |
| "clip_ratio": 0.0002561475266702473, |
| "epoch": 0.09270890223921355, |
| "grad_norm": 0.02690211683511734, |
| "kl": 0.0018389189117442584, |
| "learning_rate": 1.6894736842105264e-06, |
| "loss": -0.0, |
| "step": 679 |
| }, |
| { |
| "clip_ratio": 0.0002561475266702473, |
| "epoch": 0.09284543965046423, |
| "grad_norm": 0.027215635403990746, |
| "kl": 0.001933387746248627, |
| "learning_rate": 1.6842105263157895e-06, |
| "loss": 0.0, |
| "step": 680 |
| }, |
| { |
| "clip_ratio": 0.000294811325147748, |
| "completion_length": 121.09375, |
| "epoch": 0.0929819770617149, |
| "grad_norm": 0.029706580564379692, |
| "kl": 0.002037748434304376, |
| "learning_rate": 1.6789473684210527e-06, |
| "loss": 0.0, |
| "num_tokens": 1986749.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 681 |
| }, |
| { |
| "clip_ratio": 0.000294811325147748, |
| "epoch": 0.0931185144729656, |
| "grad_norm": 0.02914939634501934, |
| "kl": 0.0020570547858369537, |
| "learning_rate": 1.673684210526316e-06, |
| "loss": 0.0001, |
| "step": 682 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09325505188421628, |
| "grad_norm": 0.02946673519909382, |
| "kl": 0.002120211442161235, |
| "learning_rate": 1.6684210526315792e-06, |
| "loss": 0.0, |
| "step": 683 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09339158929546695, |
| "grad_norm": 0.0294260922819376, |
| "kl": 0.0021350198639993323, |
| "learning_rate": 1.6631578947368424e-06, |
| "loss": -0.0001, |
| "step": 684 |
| }, |
| { |
| "clip_ratio": 0.00019409938249737024, |
| "completion_length": 149.40625, |
| "epoch": 0.09352812670671765, |
| "grad_norm": 0.024748150259256363, |
| "kl": 0.001692789483058732, |
| "learning_rate": 1.6578947368421053e-06, |
| "loss": -0.0, |
| "num_tokens": 1998122.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 685 |
| }, |
| { |
| "clip_ratio": 0.0007354209810728207, |
| "epoch": 0.09366466411796832, |
| "grad_norm": 0.025075767189264297, |
| "kl": 0.0016773628740338609, |
| "learning_rate": 1.6526315789473685e-06, |
| "loss": 0.0, |
| "step": 686 |
| }, |
| { |
| "clip_ratio": 0.0005357511981856078, |
| "epoch": 0.093801201529219, |
| "grad_norm": 0.024616461247205734, |
| "kl": 0.001673653707257472, |
| "learning_rate": 1.6473684210526317e-06, |
| "loss": -0.0, |
| "step": 687 |
| }, |
| { |
| "clip_ratio": 0.00012400794366840273, |
| "epoch": 0.0939377389404697, |
| "grad_norm": 0.025905825197696686, |
| "kl": 0.0016978087533061625, |
| "learning_rate": 1.6421052631578948e-06, |
| "loss": 0.0, |
| "step": 688 |
| }, |
| { |
| "clip_ratio": 0.0005619380681309849, |
| "completion_length": 134.65625, |
| "epoch": 0.09407427635172037, |
| "grad_norm": 0.031999699771404266, |
| "kl": 0.0016089139571704436, |
| "learning_rate": 1.636842105263158e-06, |
| "loss": -0.0001, |
| "num_tokens": 2010007.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 689 |
| }, |
| { |
| "clip_ratio": 0.00034340660204179585, |
| "epoch": 0.09421081376297105, |
| "grad_norm": 0.03210426867008209, |
| "kl": 0.0015337139411712997, |
| "learning_rate": 1.6315789473684212e-06, |
| "loss": 0.0001, |
| "step": 690 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09434735117422173, |
| "grad_norm": 0.03191930055618286, |
| "kl": 0.001649985551921418, |
| "learning_rate": 1.6263157894736845e-06, |
| "loss": -0.0002, |
| "step": 691 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09448388858547242, |
| "grad_norm": 0.0321015864610672, |
| "kl": 0.0016289945742755663, |
| "learning_rate": 1.6210526315789473e-06, |
| "loss": 0.0001, |
| "step": 692 |
| }, |
| { |
| "clip_ratio": 0.0004166830622125417, |
| "completion_length": 119.09375, |
| "epoch": 0.0946204259967231, |
| "grad_norm": 0.027585109695792198, |
| "kl": 0.0017405743255949346, |
| "learning_rate": 1.6157894736842106e-06, |
| "loss": 0.0, |
| "num_tokens": 2021430.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 693 |
| }, |
| { |
| "clip_ratio": 0.0006837770924903452, |
| "epoch": 0.09475696340797378, |
| "grad_norm": 0.02802807278931141, |
| "kl": 0.0017661270230746595, |
| "learning_rate": 1.6105263157894738e-06, |
| "loss": 0.0002, |
| "step": 694 |
| }, |
| { |
| "clip_ratio": 0.0004166830622125417, |
| "epoch": 0.09489350081922447, |
| "grad_norm": 0.02761889435350895, |
| "kl": 0.0017125707254308509, |
| "learning_rate": 1.605263157894737e-06, |
| "loss": -0.0, |
| "step": 695 |
| }, |
| { |
| "clip_ratio": 0.0008214422850869596, |
| "epoch": 0.09503003823047515, |
| "grad_norm": 0.027634145691990852, |
| "kl": 0.0018781941034831107, |
| "learning_rate": 1.6000000000000001e-06, |
| "loss": -0.0002, |
| "step": 696 |
| }, |
| { |
| "clip_ratio": 0.0011033393384423107, |
| "completion_length": 138.78125, |
| "epoch": 0.09516657564172583, |
| "grad_norm": 0.03645732253789902, |
| "kl": 0.0015163161588134244, |
| "learning_rate": 1.5947368421052633e-06, |
| "loss": 0.0, |
| "num_tokens": 2033031.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.298653781414032, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 697 |
| }, |
| { |
| "clip_ratio": 0.00014076576917432249, |
| "epoch": 0.09530311305297652, |
| "grad_norm": 0.035988595336675644, |
| "kl": 0.0015767957502248464, |
| "learning_rate": 1.5894736842105265e-06, |
| "loss": -0.0001, |
| "step": 698 |
| }, |
| { |
| "clip_ratio": 0.0010361238601035438, |
| "epoch": 0.0954396504642272, |
| "grad_norm": 0.03622249513864517, |
| "kl": 0.0015686019669374218, |
| "learning_rate": 1.5842105263157894e-06, |
| "loss": -0.0001, |
| "step": 699 |
| }, |
| { |
| "clip_ratio": 0.00037931538827251643, |
| "epoch": 0.09557618787547788, |
| "grad_norm": 0.03611570969223976, |
| "kl": 0.0015960140808601864, |
| "learning_rate": 1.5789473684210526e-06, |
| "loss": -0.0003, |
| "step": 700 |
| }, |
| { |
| "clip_ratio": 0.0005520998747670092, |
| "completion_length": 161.96875, |
| "epoch": 0.09571272528672857, |
| "grad_norm": 0.03334690257906914, |
| "kl": 0.0014301782102847937, |
| "learning_rate": 1.573684210526316e-06, |
| "loss": -0.0001, |
| "num_tokens": 2046202.0, |
| "reward": 0.40625, |
| "reward_std": 0.3049038052558899, |
| "rewards/reward_fn": 0.40625, |
| "step": 701 |
| }, |
| { |
| "clip_ratio": 0.00041565170977264643, |
| "epoch": 0.09584926269797925, |
| "grad_norm": 0.03214196488261223, |
| "kl": 0.001410563058925618, |
| "learning_rate": 1.5684210526315791e-06, |
| "loss": -0.0002, |
| "step": 702 |
| }, |
| { |
| "clip_ratio": 0.000545183866051957, |
| "epoch": 0.09598580010922993, |
| "grad_norm": 0.033150359988212585, |
| "kl": 0.0013807993827867904, |
| "learning_rate": 1.5631578947368423e-06, |
| "loss": 0.0001, |
| "step": 703 |
| }, |
| { |
| "clip_ratio": 0.00011160714348079637, |
| "epoch": 0.0961223375204806, |
| "grad_norm": 0.032138846814632416, |
| "kl": 0.0013679018711627577, |
| "learning_rate": 1.5578947368421054e-06, |
| "loss": -0.0, |
| "step": 704 |
| }, |
| { |
| "clip_ratio": 0.0004139072843827307, |
| "completion_length": 129.375, |
| "epoch": 0.0962588749317313, |
| "grad_norm": 0.03673640638589859, |
| "kl": 0.0017190880898851901, |
| "learning_rate": 1.5526315789473686e-06, |
| "loss": 0.0001, |
| "num_tokens": 2057270.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 705 |
| }, |
| { |
| "clip_ratio": 0.00036971406370867044, |
| "epoch": 0.09639541234298198, |
| "grad_norm": 0.034445621073246, |
| "kl": 0.001723575060168514, |
| "learning_rate": 1.5473684210526316e-06, |
| "loss": 0.0001, |
| "step": 706 |
| }, |
| { |
| "clip_ratio": 0.0008860736415954307, |
| "epoch": 0.09653194975423265, |
| "grad_norm": 0.036579687148332596, |
| "kl": 0.0016895698117878055, |
| "learning_rate": 1.5421052631578947e-06, |
| "loss": -0.0002, |
| "step": 707 |
| }, |
| { |
| "clip_ratio": 0.00046615848259534687, |
| "epoch": 0.09666848716548335, |
| "grad_norm": 0.036325111985206604, |
| "kl": 0.0016936773936322425, |
| "learning_rate": 1.5368421052631579e-06, |
| "loss": -0.0002, |
| "step": 708 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 127.28125, |
| "epoch": 0.09680502457673403, |
| "grad_norm": 0.02210831828415394, |
| "kl": 0.0017170392002299195, |
| "learning_rate": 1.5315789473684213e-06, |
| "loss": -0.0001, |
| "num_tokens": 2068215.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 709 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0969415619879847, |
| "grad_norm": 0.02184915728867054, |
| "kl": 0.0016970243068499258, |
| "learning_rate": 1.5263157894736844e-06, |
| "loss": -0.0001, |
| "step": 710 |
| }, |
| { |
| "clip_ratio": 0.0001594387722434476, |
| "epoch": 0.0970780993992354, |
| "grad_norm": 0.022008102387189865, |
| "kl": 0.001666097336965322, |
| "learning_rate": 1.5210526315789476e-06, |
| "loss": -0.0, |
| "step": 711 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09721463681048607, |
| "grad_norm": 0.02160630375146866, |
| "kl": 0.0016568601295148255, |
| "learning_rate": 1.5157894736842108e-06, |
| "loss": -0.0, |
| "step": 712 |
| }, |
| { |
| "clip_ratio": 0.0008411372982664034, |
| "completion_length": 149.9375, |
| "epoch": 0.09735117422173675, |
| "grad_norm": 0.03211577236652374, |
| "kl": 0.0015268612896761624, |
| "learning_rate": 1.5105263157894737e-06, |
| "loss": -0.0003, |
| "num_tokens": 2080137.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2511056959629059, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 713 |
| }, |
| { |
| "clip_ratio": 0.0008598162676207721, |
| "epoch": 0.09748771163298744, |
| "grad_norm": 0.03194946050643921, |
| "kl": 0.0014656807561550522, |
| "learning_rate": 1.5052631578947369e-06, |
| "loss": -0.0001, |
| "step": 714 |
| }, |
| { |
| "clip_ratio": 0.0011155456595588475, |
| "epoch": 0.09762424904423812, |
| "grad_norm": 0.03199795261025429, |
| "kl": 0.0015198987184703583, |
| "learning_rate": 1.5e-06, |
| "loss": -0.0001, |
| "step": 715 |
| }, |
| { |
| "clip_ratio": 0.0004340070008765906, |
| "epoch": 0.0977607864554888, |
| "grad_norm": 0.03166012093424797, |
| "kl": 0.0015317311590479221, |
| "learning_rate": 1.4947368421052632e-06, |
| "loss": -0.0002, |
| "step": 716 |
| }, |
| { |
| "clip_ratio": 0.0010012474449467845, |
| "completion_length": 150.5625, |
| "epoch": 0.09789732386673948, |
| "grad_norm": 0.033033497631549835, |
| "kl": 0.0016807273759695818, |
| "learning_rate": 1.4894736842105264e-06, |
| "loss": -0.0002, |
| "num_tokens": 2092099.0, |
| "reward": 0.859375, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.859375, |
| "step": 717 |
| }, |
| { |
| "clip_ratio": 0.000548348885786254, |
| "epoch": 0.09803386127799017, |
| "grad_norm": 0.033237796276807785, |
| "kl": 0.0017241216792172054, |
| "learning_rate": 1.4842105263157897e-06, |
| "loss": -0.0, |
| "step": 718 |
| }, |
| { |
| "clip_ratio": 0.00046202958037611097, |
| "epoch": 0.09817039868924085, |
| "grad_norm": 0.03423339128494263, |
| "kl": 0.0016696170132490806, |
| "learning_rate": 1.478947368421053e-06, |
| "loss": -0.0001, |
| "step": 719 |
| }, |
| { |
| "clip_ratio": 0.0008547664401703514, |
| "epoch": 0.09830693610049153, |
| "grad_norm": 0.03277941793203354, |
| "kl": 0.0016932836006162688, |
| "learning_rate": 1.4736842105263159e-06, |
| "loss": -0.0001, |
| "step": 720 |
| }, |
| { |
| "clip_ratio": 0.0007789192459313199, |
| "completion_length": 126.40625, |
| "epoch": 0.09844347351174222, |
| "grad_norm": 0.02932615391910076, |
| "kl": 0.0018991009465025854, |
| "learning_rate": 1.468421052631579e-06, |
| "loss": -0.0, |
| "num_tokens": 2103080.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 721 |
| }, |
| { |
| "clip_ratio": 0.00021701389050576836, |
| "epoch": 0.0985800109229929, |
| "grad_norm": 0.02889619581401348, |
| "kl": 0.0018387689356131887, |
| "learning_rate": 1.4631578947368422e-06, |
| "loss": 0.0001, |
| "step": 722 |
| }, |
| { |
| "clip_ratio": 0.0009659523202572018, |
| "epoch": 0.09871654833424358, |
| "grad_norm": 0.029005665332078934, |
| "kl": 0.0018426774186082184, |
| "learning_rate": 1.4578947368421053e-06, |
| "loss": 0.0001, |
| "step": 723 |
| }, |
| { |
| "clip_ratio": 0.0007789192459313199, |
| "epoch": 0.09885308574549427, |
| "grad_norm": 0.029167210683226585, |
| "kl": 0.001885860742731893, |
| "learning_rate": 1.4526315789473685e-06, |
| "loss": -0.0001, |
| "step": 724 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.65625, |
| "epoch": 0.09898962315674495, |
| "grad_norm": 0.036384548991918564, |
| "kl": 0.0014372121031556162, |
| "learning_rate": 1.4473684210526317e-06, |
| "loss": 0.0001, |
| "num_tokens": 2114013.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.2511056959629059, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 725 |
| }, |
| { |
| "clip_ratio": 0.00023854961909819394, |
| "epoch": 0.09912616056799563, |
| "grad_norm": 0.036850765347480774, |
| "kl": 0.0016032819266911247, |
| "learning_rate": 1.442105263157895e-06, |
| "loss": -0.0, |
| "step": 726 |
| }, |
| { |
| "clip_ratio": 0.0003396739193703979, |
| "epoch": 0.09926269797924632, |
| "grad_norm": 0.0372999869287014, |
| "kl": 0.0015850669196879608, |
| "learning_rate": 1.4368421052631582e-06, |
| "loss": 0.0001, |
| "step": 727 |
| }, |
| { |
| "clip_ratio": 0.00021853146608918905, |
| "epoch": 0.099399235390497, |
| "grad_norm": 0.03686143085360527, |
| "kl": 0.001440588292098255, |
| "learning_rate": 1.4315789473684212e-06, |
| "loss": 0.0, |
| "step": 728 |
| }, |
| { |
| "clip_ratio": 0.00048737886390881613, |
| "completion_length": 144.4375, |
| "epoch": 0.09953577280174768, |
| "grad_norm": 0.03900820016860962, |
| "kl": 0.0018115455322913476, |
| "learning_rate": 1.4263157894736843e-06, |
| "loss": -0.0002, |
| "num_tokens": 2125843.0, |
| "reward": 0.71875, |
| "reward_std": 0.35490378737449646, |
| "rewards/reward_fn": 0.71875, |
| "step": 729 |
| }, |
| { |
| "clip_ratio": 0.0004231298589729704, |
| "epoch": 0.09967231021299836, |
| "grad_norm": 0.03907107934355736, |
| "kl": 0.0018483502553863218, |
| "learning_rate": 1.4210526315789475e-06, |
| "loss": -0.0002, |
| "step": 730 |
| }, |
| { |
| "clip_ratio": 0.0006401437494787388, |
| "epoch": 0.09980884762424905, |
| "grad_norm": 0.03969133272767067, |
| "kl": 0.001942682627486647, |
| "learning_rate": 1.4157894736842107e-06, |
| "loss": -0.0, |
| "step": 731 |
| }, |
| { |
| "clip_ratio": 0.0005634278713841923, |
| "epoch": 0.09994538503549973, |
| "grad_norm": 0.03949359059333801, |
| "kl": 0.0019668521690618945, |
| "learning_rate": 1.4105263157894738e-06, |
| "loss": -0.0002, |
| "step": 732 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 135.09375, |
| "epoch": 0.1000819224467504, |
| "grad_norm": 0.012566876597702503, |
| "kl": 0.0020119995751883835, |
| "learning_rate": 1.405263157894737e-06, |
| "loss": -0.0, |
| "num_tokens": 2137698.0, |
| "reward": 0.859375, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.859375, |
| "step": 733 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1002184598580011, |
| "grad_norm": 0.017249109223484993, |
| "kl": 0.0017891458755912026, |
| "learning_rate": 1.4000000000000001e-06, |
| "loss": -0.0001, |
| "step": 734 |
| }, |
| { |
| "clip_ratio": 0.0001849112450145185, |
| "epoch": 0.10035499726925177, |
| "grad_norm": 0.017133330926299095, |
| "kl": 0.001893964958981087, |
| "learning_rate": 1.394736842105263e-06, |
| "loss": -0.0, |
| "step": 735 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10049153468050245, |
| "grad_norm": 0.017315877601504326, |
| "kl": 0.0018965235340147046, |
| "learning_rate": 1.3894736842105263e-06, |
| "loss": -0.0001, |
| "step": 736 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.625, |
| "epoch": 0.10062807209175315, |
| "grad_norm": 0.02570240944623947, |
| "kl": 0.002060361744952388, |
| "learning_rate": 1.3842105263157896e-06, |
| "loss": 0.0, |
| "num_tokens": 2149018.0, |
| "reward": 0.859375, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.859375, |
| "step": 737 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "epoch": 0.10076460950300382, |
| "grad_norm": 0.026232900097966194, |
| "kl": 0.0021926913996139774, |
| "learning_rate": 1.3789473684210528e-06, |
| "loss": -0.0, |
| "step": 738 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "epoch": 0.1009011469142545, |
| "grad_norm": 0.025098463520407677, |
| "kl": 0.002106565338181099, |
| "learning_rate": 1.373684210526316e-06, |
| "loss": -0.0002, |
| "step": 739 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "epoch": 0.1010376843255052, |
| "grad_norm": 0.025355098769068718, |
| "kl": 0.0020467557496886, |
| "learning_rate": 1.3684210526315791e-06, |
| "loss": 0.0002, |
| "step": 740 |
| }, |
| { |
| "clip_ratio": 0.00098407655605115, |
| "completion_length": 132.4375, |
| "epoch": 0.10117422173675587, |
| "grad_norm": 0.036574024707078934, |
| "kl": 0.0018209284698968986, |
| "learning_rate": 1.3631578947368423e-06, |
| "loss": 0.0, |
| "num_tokens": 2160252.0, |
| "reward": 0.4937499761581421, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.4937499761581421, |
| "step": 741 |
| }, |
| { |
| "clip_ratio": 0.0007863230566726997, |
| "epoch": 0.10131075914800655, |
| "grad_norm": 0.03622036427259445, |
| "kl": 0.0018895867688115686, |
| "learning_rate": 1.3578947368421052e-06, |
| "loss": -0.0001, |
| "step": 742 |
| }, |
| { |
| "clip_ratio": 0.0010683000582503155, |
| "epoch": 0.10144729655925723, |
| "grad_norm": 0.03665731102228165, |
| "kl": 0.0018325582204852253, |
| "learning_rate": 1.3526315789473684e-06, |
| "loss": -0.0001, |
| "step": 743 |
| }, |
| { |
| "clip_ratio": 0.0008244863129220903, |
| "epoch": 0.10158383397050792, |
| "grad_norm": 0.036230310797691345, |
| "kl": 0.0018342306175327394, |
| "learning_rate": 1.3473684210526316e-06, |
| "loss": -0.0002, |
| "step": 744 |
| }, |
| { |
| "clip_ratio": 0.00041680221329443157, |
| "completion_length": 141.625, |
| "epoch": 0.1017203713817586, |
| "grad_norm": 0.03272077441215515, |
| "kl": 0.0017692799974611262, |
| "learning_rate": 1.342105263157895e-06, |
| "loss": -0.0001, |
| "num_tokens": 2171828.0, |
| "reward": 0.4937500059604645, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.4937500059604645, |
| "step": 745 |
| }, |
| { |
| "clip_ratio": 0.000294811325147748, |
| "epoch": 0.10185690879300928, |
| "grad_norm": 0.03340604528784752, |
| "kl": 0.0016943258451647125, |
| "learning_rate": 1.3368421052631581e-06, |
| "loss": 0.0001, |
| "step": 746 |
| }, |
| { |
| "clip_ratio": 0.0011052459594793618, |
| "epoch": 0.10199344620425997, |
| "grad_norm": 0.03306342288851738, |
| "kl": 0.0018010039475484518, |
| "learning_rate": 1.3315789473684213e-06, |
| "loss": 0.0, |
| "step": 747 |
| }, |
| { |
| "clip_ratio": 0.00026939655072055757, |
| "epoch": 0.10212998361551065, |
| "grad_norm": 0.032833609730005264, |
| "kl": 0.0018719100971793523, |
| "learning_rate": 1.3263157894736844e-06, |
| "loss": 0.0003, |
| "step": 748 |
| }, |
| { |
| "clip_ratio": 0.0009049124200828373, |
| "completion_length": 122.84375, |
| "epoch": 0.10226652102676133, |
| "grad_norm": 0.027507351711392403, |
| "kl": 0.0014249746527639218, |
| "learning_rate": 1.3210526315789474e-06, |
| "loss": -0.0001, |
| "num_tokens": 2183135.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 749 |
| }, |
| { |
| "clip_ratio": 0.0008354411402251571, |
| "epoch": 0.10240305843801202, |
| "grad_norm": 0.02685280330479145, |
| "kl": 0.0016018145142879803, |
| "learning_rate": 1.3157894736842106e-06, |
| "loss": 0.0001, |
| "step": 750 |
| }, |
| { |
| "clip_ratio": 0.0004125279519939795, |
| "epoch": 0.1025395958492627, |
| "grad_norm": 0.02706029638648033, |
| "kl": 0.0015976888844306814, |
| "learning_rate": 1.3105263157894737e-06, |
| "loss": -0.0, |
| "step": 751 |
| }, |
| { |
| "clip_ratio": 0.0006423073646146804, |
| "epoch": 0.10267613326051338, |
| "grad_norm": 0.027195792645215988, |
| "kl": 0.001565335212944774, |
| "learning_rate": 1.3052631578947369e-06, |
| "loss": -0.0001, |
| "step": 752 |
| }, |
| { |
| "clip_ratio": 0.00047186609299387783, |
| "completion_length": 125.5, |
| "epoch": 0.10281267067176407, |
| "grad_norm": 0.04647589102387428, |
| "kl": 0.0022257442942645866, |
| "learning_rate": 1.3e-06, |
| "loss": -0.0003, |
| "num_tokens": 2194823.0, |
| "reward": 0.49375003576278687, |
| "reward_std": 0.37230759859085083, |
| "rewards/reward_fn": 0.49375003576278687, |
| "step": 753 |
| }, |
| { |
| "clip_ratio": 0.00031565656536258757, |
| "epoch": 0.10294920808301475, |
| "grad_norm": 0.046547677367925644, |
| "kl": 0.00213234518741956, |
| "learning_rate": 1.2947368421052634e-06, |
| "loss": 0.0, |
| "step": 754 |
| }, |
| { |
| "clip_ratio": 0.0009570495167281479, |
| "epoch": 0.10308574549426543, |
| "grad_norm": 0.046105701476335526, |
| "kl": 0.002177114802179858, |
| "learning_rate": 1.2894736842105266e-06, |
| "loss": -0.0001, |
| "step": 755 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1032222829055161, |
| "grad_norm": 0.04649042338132858, |
| "kl": 0.002199585574999219, |
| "learning_rate": 1.2842105263157895e-06, |
| "loss": -0.0004, |
| "step": 756 |
| }, |
| { |
| "clip_ratio": 0.00028935185400769114, |
| "completion_length": 131.09375, |
| "epoch": 0.1033588203167668, |
| "grad_norm": 0.032286182045936584, |
| "kl": 0.0018966078678204212, |
| "learning_rate": 1.2789473684210527e-06, |
| "loss": -0.0001, |
| "num_tokens": 2206574.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 757 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10349535772801748, |
| "grad_norm": 0.031949903815984726, |
| "kl": 0.001969371132872766, |
| "learning_rate": 1.2736842105263159e-06, |
| "loss": -0.0002, |
| "step": 758 |
| }, |
| { |
| "clip_ratio": 0.00028935185400769114, |
| "epoch": 0.10363189513926815, |
| "grad_norm": 0.03214770555496216, |
| "kl": 0.0018590922591101844, |
| "learning_rate": 1.268421052631579e-06, |
| "loss": 0.0002, |
| "step": 759 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10376843255051885, |
| "grad_norm": 0.041023969650268555, |
| "kl": 0.0018697814284678316, |
| "learning_rate": 1.2631578947368422e-06, |
| "loss": 0.0, |
| "step": 760 |
| }, |
| { |
| "clip_ratio": 0.00035995818325318396, |
| "completion_length": 137.375, |
| "epoch": 0.10390496996176952, |
| "grad_norm": 0.03233378753066063, |
| "kl": 0.0022407320539059583, |
| "learning_rate": 1.2578947368421054e-06, |
| "loss": -0.0, |
| "num_tokens": 2218178.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.578125, |
| "step": 761 |
| }, |
| { |
| "clip_ratio": 0.00014880952949170023, |
| "epoch": 0.1040415073730202, |
| "grad_norm": 0.03180888295173645, |
| "kl": 0.0023300516895687906, |
| "learning_rate": 1.2526315789473687e-06, |
| "loss": -0.0001, |
| "step": 762 |
| }, |
| { |
| "clip_ratio": 0.00021114865376148373, |
| "epoch": 0.1041780447842709, |
| "grad_norm": 0.03168882802128792, |
| "kl": 0.0021711552963097347, |
| "learning_rate": 1.2473684210526317e-06, |
| "loss": -0.0002, |
| "step": 763 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10431458219552157, |
| "grad_norm": 0.03188063204288483, |
| "kl": 0.002269363276354852, |
| "learning_rate": 1.2421052631578948e-06, |
| "loss": -0.0001, |
| "step": 764 |
| }, |
| { |
| "clip_ratio": 0.0005279374017845839, |
| "completion_length": 134.09375, |
| "epoch": 0.10445111960677225, |
| "grad_norm": 0.025516286492347717, |
| "kl": 0.0019899348426406505, |
| "learning_rate": 1.236842105263158e-06, |
| "loss": 0.0001, |
| "num_tokens": 2229037.0, |
| "reward": 0.9437500238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.9437500238418579, |
| "step": 765 |
| }, |
| { |
| "clip_ratio": 0.00016622339899186045, |
| "epoch": 0.10458765701802294, |
| "grad_norm": 0.025222698226571083, |
| "kl": 0.001958257113074069, |
| "learning_rate": 1.2315789473684212e-06, |
| "loss": -0.0, |
| "step": 766 |
| }, |
| { |
| "clip_ratio": 0.0006586723029613495, |
| "epoch": 0.10472419442927362, |
| "grad_norm": 0.02710876613855362, |
| "kl": 0.0018938343164336402, |
| "learning_rate": 1.2263157894736843e-06, |
| "loss": 0.0001, |
| "step": 767 |
| }, |
| { |
| "clip_ratio": 0.00021853146608918905, |
| "epoch": 0.1048607318405243, |
| "grad_norm": 0.025685269385576248, |
| "kl": 0.0020639228823711164, |
| "learning_rate": 1.2210526315789475e-06, |
| "loss": -0.0, |
| "step": 768 |
| }, |
| { |
| "clip_ratio": 0.0011217906430829316, |
| "completion_length": 133.96875, |
| "epoch": 0.10499726925177498, |
| "grad_norm": 0.0428641252219677, |
| "kl": 0.0020228277189744404, |
| "learning_rate": 1.2157894736842107e-06, |
| "loss": 0.0003, |
| "num_tokens": 2240432.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 769 |
| }, |
| { |
| "clip_ratio": 0.0015004310989752412, |
| "epoch": 0.10513380666302567, |
| "grad_norm": 0.04304884001612663, |
| "kl": 0.0020226746928528883, |
| "learning_rate": 1.2105263157894738e-06, |
| "loss": -0.0001, |
| "step": 770 |
| }, |
| { |
| "clip_ratio": 0.0010420014732517302, |
| "epoch": 0.10527034407427635, |
| "grad_norm": 0.04133768379688263, |
| "kl": 0.001905648708998342, |
| "learning_rate": 1.205263157894737e-06, |
| "loss": 0.0001, |
| "step": 771 |
| }, |
| { |
| "clip_ratio": 0.0007046800164971501, |
| "epoch": 0.10540688148552703, |
| "grad_norm": 0.04461239278316498, |
| "kl": 0.0019738287810469046, |
| "learning_rate": 1.2000000000000002e-06, |
| "loss": 0.0002, |
| "step": 772 |
| }, |
| { |
| "clip_ratio": 0.00048007245641201735, |
| "completion_length": 123.4375, |
| "epoch": 0.10554341889677772, |
| "grad_norm": 0.03209725022315979, |
| "kl": 0.002003139950829791, |
| "learning_rate": 1.1947368421052633e-06, |
| "loss": -0.0, |
| "num_tokens": 2251734.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 773 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1056799563080284, |
| "grad_norm": 0.032378919422626495, |
| "kl": 0.0019621488609118387, |
| "learning_rate": 1.1894736842105265e-06, |
| "loss": 0.0, |
| "step": 774 |
| }, |
| { |
| "clip_ratio": 0.000294811325147748, |
| "epoch": 0.10581649371927908, |
| "grad_norm": 0.032106876373291016, |
| "kl": 0.001987692747206893, |
| "learning_rate": 1.1842105263157894e-06, |
| "loss": -0.0001, |
| "step": 775 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10595303113052977, |
| "grad_norm": 0.031922806054353714, |
| "kl": 0.0019523974406183697, |
| "learning_rate": 1.1789473684210526e-06, |
| "loss": -0.0001, |
| "step": 776 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 120.5, |
| "epoch": 0.10608956854178045, |
| "grad_norm": 0.02806154638528824, |
| "kl": 0.0022280882021732396, |
| "learning_rate": 1.173684210526316e-06, |
| "loss": -0.0002, |
| "num_tokens": 2262646.0, |
| "reward": 0.578125, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.578125, |
| "step": 777 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10622610595303113, |
| "grad_norm": 0.027715560048818588, |
| "kl": 0.0022063598844397347, |
| "learning_rate": 1.1684210526315791e-06, |
| "loss": -0.0001, |
| "step": 778 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10636264336428182, |
| "grad_norm": 0.027679534628987312, |
| "kl": 0.002072686085739406, |
| "learning_rate": 1.163157894736842e-06, |
| "loss": -0.0001, |
| "step": 779 |
| }, |
| { |
| "clip_ratio": 0.000240384615608491, |
| "epoch": 0.1064991807755325, |
| "grad_norm": 0.026711298152804375, |
| "kl": 0.002109782257321058, |
| "learning_rate": 1.1578947368421053e-06, |
| "loss": -0.0, |
| "step": 780 |
| }, |
| { |
| "clip_ratio": 0.0003037487913388759, |
| "completion_length": 150.71875, |
| "epoch": 0.10663571818678318, |
| "grad_norm": 0.03064105659723282, |
| "kl": 0.001606904103027773, |
| "learning_rate": 1.1526315789473686e-06, |
| "loss": -0.0001, |
| "num_tokens": 2275025.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 781 |
| }, |
| { |
| "clip_ratio": 0.0008279759786091745, |
| "epoch": 0.10677225559803386, |
| "grad_norm": 0.03059159778058529, |
| "kl": 0.0015235491227940656, |
| "learning_rate": 1.1473684210526316e-06, |
| "loss": -0.0001, |
| "step": 782 |
| }, |
| { |
| "clip_ratio": 0.0013057087053311989, |
| "epoch": 0.10690879300928455, |
| "grad_norm": 0.029724495485424995, |
| "kl": 0.0015655803872505203, |
| "learning_rate": 1.1421052631578947e-06, |
| "loss": -0.0002, |
| "step": 783 |
| }, |
| { |
| "clip_ratio": 0.0009672492742538452, |
| "epoch": 0.10704533042053523, |
| "grad_norm": 0.030576549470424652, |
| "kl": 0.001495385740781785, |
| "learning_rate": 1.136842105263158e-06, |
| "loss": 0.0001, |
| "step": 784 |
| }, |
| { |
| "clip_ratio": 0.0011856760102091357, |
| "completion_length": 128.90625, |
| "epoch": 0.1071818678317859, |
| "grad_norm": 0.04623905569314957, |
| "kl": 0.0020615452212950913, |
| "learning_rate": 1.1315789473684213e-06, |
| "loss": 0.0001, |
| "num_tokens": 2286446.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 785 |
| }, |
| { |
| "clip_ratio": 0.000777867971919477, |
| "epoch": 0.1073184052430366, |
| "grad_norm": 0.046840786933898926, |
| "kl": 0.0020759970630024327, |
| "learning_rate": 1.1263157894736842e-06, |
| "loss": -0.0001, |
| "step": 786 |
| }, |
| { |
| "clip_ratio": 0.0015322749968618155, |
| "epoch": 0.10745494265428727, |
| "grad_norm": 0.04625287652015686, |
| "kl": 0.002053738779068226, |
| "learning_rate": 1.1210526315789474e-06, |
| "loss": 0.0002, |
| "step": 787 |
| }, |
| { |
| "clip_ratio": 0.0009743925911607221, |
| "epoch": 0.10759148006553795, |
| "grad_norm": 0.04577214643359184, |
| "kl": 0.001995397404243704, |
| "learning_rate": 1.1157894736842106e-06, |
| "loss": 0.0001, |
| "step": 788 |
| }, |
| { |
| "clip_ratio": 0.000294811325147748, |
| "completion_length": 123.875, |
| "epoch": 0.10772801747678865, |
| "grad_norm": 0.02258908562362194, |
| "kl": 0.002167003383874544, |
| "learning_rate": 1.110526315789474e-06, |
| "loss": 0.0, |
| "num_tokens": 2296770.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 789 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10786455488803932, |
| "grad_norm": 0.023535488173365593, |
| "kl": 0.001987856176128844, |
| "learning_rate": 1.1052631578947369e-06, |
| "loss": 0.0001, |
| "step": 790 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10800109229929, |
| "grad_norm": 0.023333881050348282, |
| "kl": 0.002119716620654799, |
| "learning_rate": 1.1e-06, |
| "loss": 0.0, |
| "step": 791 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1081376297105407, |
| "grad_norm": 0.024103818461298943, |
| "kl": 0.00215182734427799, |
| "learning_rate": 1.0947368421052632e-06, |
| "loss": 0.0001, |
| "step": 792 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 129.96875, |
| "epoch": 0.10827416712179137, |
| "grad_norm": 0.02682146430015564, |
| "kl": 0.0017019015285768546, |
| "learning_rate": 1.0894736842105264e-06, |
| "loss": 0.0001, |
| "num_tokens": 2308301.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 793 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10841070453304205, |
| "grad_norm": 0.0268485676497221, |
| "kl": 0.0017056646174751222, |
| "learning_rate": 1.0842105263157895e-06, |
| "loss": 0.0001, |
| "step": 794 |
| }, |
| { |
| "clip_ratio": 0.00034722223062999547, |
| "epoch": 0.10854724194429273, |
| "grad_norm": 0.027060694992542267, |
| "kl": 0.0017693005065666512, |
| "learning_rate": 1.0789473684210527e-06, |
| "loss": -0.0002, |
| "step": 795 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10868377935554342, |
| "grad_norm": 0.026868777349591255, |
| "kl": 0.0017992367465922143, |
| "learning_rate": 1.0736842105263159e-06, |
| "loss": -0.0, |
| "step": 796 |
| }, |
| { |
| "clip_ratio": 0.00020032051543239504, |
| "completion_length": 129.21875, |
| "epoch": 0.1088203167667941, |
| "grad_norm": 0.031718045473098755, |
| "kl": 0.0016662773796269903, |
| "learning_rate": 1.068421052631579e-06, |
| "loss": -0.0002, |
| "num_tokens": 2320572.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 797 |
| }, |
| { |
| "clip_ratio": 0.00032552084303461015, |
| "epoch": 0.10895685417804478, |
| "grad_norm": 0.03147350996732712, |
| "kl": 0.0020512565124590765, |
| "learning_rate": 1.0631578947368422e-06, |
| "loss": -0.0001, |
| "step": 798 |
| }, |
| { |
| "clip_ratio": 0.00032552084303461015, |
| "epoch": 0.10909339158929547, |
| "grad_norm": 0.030978526920080185, |
| "kl": 0.0017799611032387475, |
| "learning_rate": 1.0578947368421054e-06, |
| "loss": -0.0001, |
| "step": 799 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10922992900054615, |
| "grad_norm": 0.03200105205178261, |
| "kl": 0.0019068860565312207, |
| "learning_rate": 1.0526315789473685e-06, |
| "loss": -0.0001, |
| "step": 800 |
| }, |
| { |
| "clip_ratio": 0.00021701389050576836, |
| "completion_length": 133.96875, |
| "epoch": 0.10936646641179683, |
| "grad_norm": 0.030265985056757927, |
| "kl": 0.0018098962100339122, |
| "learning_rate": 1.0473684210526317e-06, |
| "loss": 0.0001, |
| "num_tokens": 2331767.0, |
| "reward": 0.578125, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.578125, |
| "step": 801 |
| }, |
| { |
| "clip_ratio": 0.00021701389050576836, |
| "epoch": 0.10950300382304752, |
| "grad_norm": 0.030390843749046326, |
| "kl": 0.0019942210747103672, |
| "learning_rate": 1.0421052631578949e-06, |
| "loss": 0.0002, |
| "step": 802 |
| }, |
| { |
| "clip_ratio": 0.0004973889590473846, |
| "epoch": 0.1096395412342982, |
| "grad_norm": 0.03022087551653385, |
| "kl": 0.0018650772599357879, |
| "learning_rate": 1.036842105263158e-06, |
| "loss": 0.0002, |
| "step": 803 |
| }, |
| { |
| "clip_ratio": 0.000717032453394495, |
| "epoch": 0.10977607864554888, |
| "grad_norm": 0.03012370690703392, |
| "kl": 0.0018872718937927857, |
| "learning_rate": 1.0315789473684212e-06, |
| "loss": 0.0002, |
| "step": 804 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 140.4375, |
| "epoch": 0.10991261605679957, |
| "grad_norm": 0.03756886348128319, |
| "kl": 0.0018428273033350706, |
| "learning_rate": 1.0263157894736843e-06, |
| "loss": -0.0001, |
| "num_tokens": 2343521.0, |
| "reward": 0.831250011920929, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.831250011920929, |
| "step": 805 |
| }, |
| { |
| "clip_ratio": 0.0007708884513704106, |
| "epoch": 0.11004915346805025, |
| "grad_norm": 0.03728393465280533, |
| "kl": 0.0018604374545247993, |
| "learning_rate": 1.0210526315789475e-06, |
| "loss": -0.0001, |
| "step": 806 |
| }, |
| { |
| "clip_ratio": 0.00040751635970082134, |
| "epoch": 0.11018569087930093, |
| "grad_norm": 0.037585336714982986, |
| "kl": 0.0019263962039985927, |
| "learning_rate": 1.0157894736842105e-06, |
| "loss": -0.0002, |
| "step": 807 |
| }, |
| { |
| "clip_ratio": 0.0006620027706958354, |
| "epoch": 0.1103222282905516, |
| "grad_norm": 0.037390291690826416, |
| "kl": 0.0018384076520305825, |
| "learning_rate": 1.0105263157894738e-06, |
| "loss": 0.0003, |
| "step": 808 |
| }, |
| { |
| "clip_ratio": 0.001664920237089973, |
| "completion_length": 135.28125, |
| "epoch": 0.1104587657018023, |
| "grad_norm": 0.0413917638361454, |
| "kl": 0.002032072839938337, |
| "learning_rate": 1.005263157894737e-06, |
| "loss": -0.0001, |
| "num_tokens": 2354734.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.35490381717681885, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 809 |
| }, |
| { |
| "clip_ratio": 0.0011410745937610045, |
| "epoch": 0.11059530311305298, |
| "grad_norm": 0.04129190742969513, |
| "kl": 0.002106687028572196, |
| "learning_rate": 1.0000000000000002e-06, |
| "loss": -0.0003, |
| "step": 810 |
| }, |
| { |
| "clip_ratio": 0.0010704625310609117, |
| "epoch": 0.11073184052430365, |
| "grad_norm": 0.041056614369153976, |
| "kl": 0.0018973293499584543, |
| "learning_rate": 9.947368421052631e-07, |
| "loss": -0.0002, |
| "step": 811 |
| }, |
| { |
| "clip_ratio": 0.001158156941528432, |
| "epoch": 0.11086837793555435, |
| "grad_norm": 0.04233892634510994, |
| "kl": 0.0020257557353033917, |
| "learning_rate": 9.894736842105265e-07, |
| "loss": -0.0002, |
| "step": 812 |
| }, |
| { |
| "clip_ratio": 0.001496172248153016, |
| "completion_length": 122.28125, |
| "epoch": 0.11100491534680502, |
| "grad_norm": 0.040789976716041565, |
| "kl": 0.002083957220747834, |
| "learning_rate": 9.842105263157897e-07, |
| "loss": -0.0001, |
| "num_tokens": 2365983.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 813 |
| }, |
| { |
| "clip_ratio": 0.0008011132304091007, |
| "epoch": 0.1111414527580557, |
| "grad_norm": 0.041051674634218216, |
| "kl": 0.0022389385812857654, |
| "learning_rate": 9.789473684210526e-07, |
| "loss": -0.0001, |
| "step": 814 |
| }, |
| { |
| "clip_ratio": 0.0021354319324018434, |
| "epoch": 0.1112779901693064, |
| "grad_norm": 0.040859155356884, |
| "kl": 0.0022529203379235696, |
| "learning_rate": 9.736842105263158e-07, |
| "loss": 0.0001, |
| "step": 815 |
| }, |
| { |
| "clip_ratio": 0.0009132857667282224, |
| "epoch": 0.11141452758055707, |
| "grad_norm": 0.04067659378051758, |
| "kl": 0.0021882897126488388, |
| "learning_rate": 9.68421052631579e-07, |
| "loss": -0.0002, |
| "step": 816 |
| }, |
| { |
| "clip_ratio": 0.0008015833009267226, |
| "completion_length": 130.875, |
| "epoch": 0.11155106499180775, |
| "grad_norm": 0.0404217429459095, |
| "kl": 0.00207597758344491, |
| "learning_rate": 9.631578947368423e-07, |
| "loss": 0.0001, |
| "num_tokens": 2377083.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 817 |
| }, |
| { |
| "clip_ratio": 0.00045197630242910236, |
| "epoch": 0.11168760240305844, |
| "grad_norm": 0.04167267680168152, |
| "kl": 0.001948569952219259, |
| "learning_rate": 9.578947368421053e-07, |
| "loss": 0.0003, |
| "step": 818 |
| }, |
| { |
| "clip_ratio": 0.0008133314258884639, |
| "epoch": 0.11182413981430912, |
| "grad_norm": 0.04034573212265968, |
| "kl": 0.00205451292640646, |
| "learning_rate": 9.526315789473685e-07, |
| "loss": 0.0004, |
| "step": 819 |
| }, |
| { |
| "clip_ratio": 0.0005783690139651299, |
| "epoch": 0.1119606772255598, |
| "grad_norm": 0.0407131165266037, |
| "kl": 0.002034339318925049, |
| "learning_rate": 9.473684210526317e-07, |
| "loss": 0.0002, |
| "step": 820 |
| }, |
| { |
| "clip_ratio": 0.0006272622267715633, |
| "completion_length": 140.40625, |
| "epoch": 0.11209721463681048, |
| "grad_norm": 0.022182300686836243, |
| "kl": 0.0017465550354245352, |
| "learning_rate": 9.421052631578948e-07, |
| "loss": -0.0002, |
| "num_tokens": 2388600.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 821 |
| }, |
| { |
| "clip_ratio": 0.0006651895891991444, |
| "epoch": 0.11223375204806117, |
| "grad_norm": 0.02211550623178482, |
| "kl": 0.001654662546570762, |
| "learning_rate": 9.368421052631579e-07, |
| "loss": -0.0002, |
| "step": 822 |
| }, |
| { |
| "clip_ratio": 0.00039533356903120875, |
| "epoch": 0.11237028945931185, |
| "grad_norm": 0.021990373730659485, |
| "kl": 0.0017915865701070288, |
| "learning_rate": 9.315789473684212e-07, |
| "loss": -0.0001, |
| "step": 823 |
| }, |
| { |
| "clip_ratio": 0.0004717895935755223, |
| "epoch": 0.11250682687056253, |
| "grad_norm": 0.022272175177931786, |
| "kl": 0.001733036981022451, |
| "learning_rate": 9.263157894736844e-07, |
| "loss": 0.0, |
| "step": 824 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 123.8125, |
| "epoch": 0.11264336428181322, |
| "grad_norm": 0.023923585191369057, |
| "kl": 0.002135575396096101, |
| "learning_rate": 9.210526315789474e-07, |
| "loss": 0.0002, |
| "num_tokens": 2400042.0, |
| "reward": 0.9156249761581421, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.9156249761581421, |
| "step": 825 |
| }, |
| { |
| "clip_ratio": 0.00015318627993110567, |
| "epoch": 0.1127799016930639, |
| "grad_norm": 0.024121804162859917, |
| "kl": 0.0022043623648642097, |
| "learning_rate": 9.157894736842106e-07, |
| "loss": 0.0001, |
| "step": 826 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11291643910431458, |
| "grad_norm": 0.02391844429075718, |
| "kl": 0.002101333857353893, |
| "learning_rate": 9.105263157894737e-07, |
| "loss": 0.0003, |
| "step": 827 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11305297651556527, |
| "grad_norm": 0.023999251425266266, |
| "kl": 0.00220367505971808, |
| "learning_rate": 9.05263157894737e-07, |
| "loss": 0.0003, |
| "step": 828 |
| }, |
| { |
| "clip_ratio": 0.0005337733018677682, |
| "completion_length": 142.71875, |
| "epoch": 0.11318951392681595, |
| "grad_norm": 0.030275406315922737, |
| "kl": 0.002487844758434221, |
| "learning_rate": 9.000000000000001e-07, |
| "loss": 0.0, |
| "num_tokens": 2411269.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 829 |
| }, |
| { |
| "clip_ratio": 0.0008539288392057642, |
| "epoch": 0.11332605133806663, |
| "grad_norm": 0.03197161853313446, |
| "kl": 0.002642420648044208, |
| "learning_rate": 8.947368421052632e-07, |
| "loss": -0.0001, |
| "step": 830 |
| }, |
| { |
| "clip_ratio": 0.0016706544120097533, |
| "epoch": 0.11346258874931732, |
| "grad_norm": 0.0321902334690094, |
| "kl": 0.002622455087475828, |
| "learning_rate": 8.894736842105264e-07, |
| "loss": -0.0001, |
| "step": 831 |
| }, |
| { |
| "clip_ratio": 0.0005035283829784021, |
| "epoch": 0.113599126160568, |
| "grad_norm": 0.031006773933768272, |
| "kl": 0.0025140510661003646, |
| "learning_rate": 8.842105263157895e-07, |
| "loss": 0.0, |
| "step": 832 |
| }, |
| { |
| "clip_ratio": 0.0008417521603405476, |
| "completion_length": 143.28125, |
| "epoch": 0.11373566357181868, |
| "grad_norm": 0.03075164556503296, |
| "kl": 0.0016546154947718605, |
| "learning_rate": 8.789473684210527e-07, |
| "loss": 0.0003, |
| "num_tokens": 2423254.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 833 |
| }, |
| { |
| "clip_ratio": 0.0014790181594435126, |
| "epoch": 0.11387220098306935, |
| "grad_norm": 0.03217291459441185, |
| "kl": 0.0017813782724260818, |
| "learning_rate": 8.736842105263159e-07, |
| "loss": 0.0001, |
| "step": 834 |
| }, |
| { |
| "clip_ratio": 0.0005646957288263366, |
| "epoch": 0.11400873839432005, |
| "grad_norm": 0.03188389539718628, |
| "kl": 0.0018060972142848186, |
| "learning_rate": 8.68421052631579e-07, |
| "loss": 0.0001, |
| "step": 835 |
| }, |
| { |
| "clip_ratio": 0.00032028641726356, |
| "epoch": 0.11414527580557073, |
| "grad_norm": 0.031433261930942535, |
| "kl": 0.0017021511393977562, |
| "learning_rate": 8.631578947368421e-07, |
| "loss": 0.0, |
| "step": 836 |
| }, |
| { |
| "clip_ratio": 0.00037020017043687403, |
| "completion_length": 131.1875, |
| "epoch": 0.1142818132168214, |
| "grad_norm": 0.026657752692699432, |
| "kl": 0.0015303445052268216, |
| "learning_rate": 8.578947368421054e-07, |
| "loss": -0.0001, |
| "num_tokens": 2434568.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 837 |
| }, |
| { |
| "clip_ratio": 0.00010888501856243238, |
| "epoch": 0.1144183506280721, |
| "grad_norm": 0.026700396090745926, |
| "kl": 0.0015061656304169446, |
| "learning_rate": 8.526315789473685e-07, |
| "loss": -0.0003, |
| "step": 838 |
| }, |
| { |
| "clip_ratio": 0.0006371819763444364, |
| "epoch": 0.11455488803932277, |
| "grad_norm": 0.026319589465856552, |
| "kl": 0.001553235877508996, |
| "learning_rate": 8.473684210526316e-07, |
| "loss": -0.0, |
| "step": 839 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11469142545057345, |
| "grad_norm": 0.027091626077890396, |
| "kl": 0.0014733337520738132, |
| "learning_rate": 8.421052631578948e-07, |
| "loss": -0.0, |
| "step": 840 |
| }, |
| { |
| "clip_ratio": 0.00023854961909819394, |
| "completion_length": 122.71875, |
| "epoch": 0.11482796286182415, |
| "grad_norm": 0.04396679624915123, |
| "kl": 0.0016556861010030843, |
| "learning_rate": 8.36842105263158e-07, |
| "loss": 0.0002, |
| "num_tokens": 2445595.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 841 |
| }, |
| { |
| "clip_ratio": 0.000645464620902203, |
| "epoch": 0.11496450027307482, |
| "grad_norm": 0.04361685737967491, |
| "kl": 0.0017527456711832201, |
| "learning_rate": 8.315789473684212e-07, |
| "loss": 0.0002, |
| "step": 842 |
| }, |
| { |
| "clip_ratio": 0.0008407137356698513, |
| "epoch": 0.1151010376843255, |
| "grad_norm": 0.044291649013757706, |
| "kl": 0.0017438465802115388, |
| "learning_rate": 8.263157894736843e-07, |
| "loss": 0.0, |
| "step": 843 |
| }, |
| { |
| "clip_ratio": 0.0006041670858394355, |
| "epoch": 0.1152375750955762, |
| "grad_norm": 0.04404996335506439, |
| "kl": 0.0018427322065690532, |
| "learning_rate": 8.210526315789474e-07, |
| "loss": 0.0001, |
| "step": 844 |
| }, |
| { |
| "clip_ratio": 0.0008153812377713621, |
| "completion_length": 140.40625, |
| "epoch": 0.11537411250682687, |
| "grad_norm": 0.03839636966586113, |
| "kl": 0.0015892905485088704, |
| "learning_rate": 8.157894736842106e-07, |
| "loss": -0.0001, |
| "num_tokens": 2457952.0, |
| "reward": 0.5499999523162842, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.5499999523162842, |
| "step": 845 |
| }, |
| { |
| "clip_ratio": 0.0012618098116945475, |
| "epoch": 0.11551064991807755, |
| "grad_norm": 0.037849295884370804, |
| "kl": 0.0015891550483502215, |
| "learning_rate": 8.105263157894736e-07, |
| "loss": -0.0002, |
| "step": 846 |
| }, |
| { |
| "clip_ratio": 0.0005228656809777021, |
| "epoch": 0.11564718732932823, |
| "grad_norm": 0.035613447427749634, |
| "kl": 0.0016328740239259787, |
| "learning_rate": 8.052631578947369e-07, |
| "loss": -0.0001, |
| "step": 847 |
| }, |
| { |
| "clip_ratio": 0.001042295160004869, |
| "epoch": 0.11578372474057892, |
| "grad_norm": 0.035603221505880356, |
| "kl": 0.0016090690733108204, |
| "learning_rate": 8.000000000000001e-07, |
| "loss": -0.0002, |
| "step": 848 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 123.9375, |
| "epoch": 0.1159202621518296, |
| "grad_norm": 0.0288266371935606, |
| "kl": 0.002268145781272324, |
| "learning_rate": 7.947368421052632e-07, |
| "loss": 0.0001, |
| "num_tokens": 2468550.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 849 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "epoch": 0.11605679956308028, |
| "grad_norm": 0.028440002351999283, |
| "kl": 0.0022938305392017355, |
| "learning_rate": 7.894736842105263e-07, |
| "loss": -0.0, |
| "step": 850 |
| }, |
| { |
| "clip_ratio": 0.00028153153834864497, |
| "epoch": 0.11619333697433097, |
| "grad_norm": 0.0285995714366436, |
| "kl": 0.002212948449596297, |
| "learning_rate": 7.842105263157896e-07, |
| "loss": -0.0001, |
| "step": 851 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "epoch": 0.11632987438558165, |
| "grad_norm": 0.02846153825521469, |
| "kl": 0.0023701565587543882, |
| "learning_rate": 7.789473684210527e-07, |
| "loss": -0.0, |
| "step": 852 |
| }, |
| { |
| "clip_ratio": 0.0001230314956046641, |
| "completion_length": 128.46875, |
| "epoch": 0.11646641179683233, |
| "grad_norm": 0.01918521709740162, |
| "kl": 0.002273245157994097, |
| "learning_rate": 7.736842105263158e-07, |
| "loss": -0.0, |
| "num_tokens": 2479605.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 853 |
| }, |
| { |
| "clip_ratio": 0.0001230314956046641, |
| "epoch": 0.11660294920808302, |
| "grad_norm": 0.019171753898262978, |
| "kl": 0.0023291846446227282, |
| "learning_rate": 7.684210526315789e-07, |
| "loss": -0.0001, |
| "step": 854 |
| }, |
| { |
| "clip_ratio": 0.0003671721206046641, |
| "epoch": 0.1167394866193337, |
| "grad_norm": 0.019308310002088547, |
| "kl": 0.0023352275111392373, |
| "learning_rate": 7.631578947368422e-07, |
| "loss": 0.0, |
| "step": 855 |
| }, |
| { |
| "clip_ratio": 0.0006592281861230731, |
| "epoch": 0.11687602403058438, |
| "grad_norm": 0.01928950659930706, |
| "kl": 0.0023274405393749475, |
| "learning_rate": 7.578947368421054e-07, |
| "loss": 0.0, |
| "step": 856 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 159.09375, |
| "epoch": 0.11701256144183507, |
| "grad_norm": 0.0285057183355093, |
| "kl": 0.0015260744385159342, |
| "learning_rate": 7.526315789473684e-07, |
| "loss": -0.0001, |
| "num_tokens": 2492020.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 857 |
| }, |
| { |
| "clip_ratio": 0.00015703517419751734, |
| "epoch": 0.11714909885308575, |
| "grad_norm": 0.02803329937160015, |
| "kl": 0.001516848760729772, |
| "learning_rate": 7.473684210526316e-07, |
| "loss": -0.0, |
| "step": 858 |
| }, |
| { |
| "clip_ratio": 0.00015703517419751734, |
| "epoch": 0.11728563626433643, |
| "grad_norm": 0.02815289795398712, |
| "kl": 0.0014452427276410162, |
| "learning_rate": 7.421052631578949e-07, |
| "loss": -0.0001, |
| "step": 859 |
| }, |
| { |
| "clip_ratio": 0.0002603958855615929, |
| "epoch": 0.1174221736755871, |
| "grad_norm": 0.02827727235853672, |
| "kl": 0.0014738957925146678, |
| "learning_rate": 7.368421052631579e-07, |
| "loss": -0.0001, |
| "step": 860 |
| }, |
| { |
| "clip_ratio": 0.00027901786961592734, |
| "completion_length": 133.28125, |
| "epoch": 0.1175587110868378, |
| "grad_norm": 0.029622992500662804, |
| "kl": 0.0017857067632576218, |
| "learning_rate": 7.315789473684211e-07, |
| "loss": 0.0001, |
| "num_tokens": 2503657.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 861 |
| }, |
| { |
| "clip_ratio": 0.000202922077733092, |
| "epoch": 0.11769524849808848, |
| "grad_norm": 0.029818588867783546, |
| "kl": 0.0018871719075832516, |
| "learning_rate": 7.263157894736843e-07, |
| "loss": -0.0001, |
| "step": 862 |
| }, |
| { |
| "clip_ratio": 0.0004327014903537929, |
| "epoch": 0.11783178590933915, |
| "grad_norm": 0.03024519979953766, |
| "kl": 0.0019561543740564957, |
| "learning_rate": 7.210526315789475e-07, |
| "loss": 0.0001, |
| "step": 863 |
| }, |
| { |
| "clip_ratio": 0.0002297794126207009, |
| "epoch": 0.11796832332058985, |
| "grad_norm": 0.029245659708976746, |
| "kl": 0.0018293752600584412, |
| "learning_rate": 7.157894736842106e-07, |
| "loss": 0.0, |
| "step": 864 |
| }, |
| { |
| "clip_ratio": 0.0004111669841222465, |
| "completion_length": 135.375, |
| "epoch": 0.11810486073184052, |
| "grad_norm": 0.0331193283200264, |
| "kl": 0.002089089653964038, |
| "learning_rate": 7.105263157894737e-07, |
| "loss": -0.0002, |
| "num_tokens": 2515349.0, |
| "reward": 0.46562501788139343, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.46562501788139343, |
| "step": 865 |
| }, |
| { |
| "clip_ratio": 0.0005714233993785456, |
| "epoch": 0.1182413981430912, |
| "grad_norm": 0.033026281744241714, |
| "kl": 0.00206513051125512, |
| "learning_rate": 7.052631578947369e-07, |
| "loss": -0.0, |
| "step": 866 |
| }, |
| { |
| "clip_ratio": 0.0005714233993785456, |
| "epoch": 0.1183779355543419, |
| "grad_norm": 0.03322478011250496, |
| "kl": 0.002053415919363033, |
| "learning_rate": 7.000000000000001e-07, |
| "loss": 0.0, |
| "step": 867 |
| }, |
| { |
| "clip_ratio": 0.0007658765680389479, |
| "epoch": 0.11851447296559257, |
| "grad_norm": 0.03364533185958862, |
| "kl": 0.002117756648658542, |
| "learning_rate": 6.947368421052631e-07, |
| "loss": -0.0, |
| "step": 868 |
| }, |
| { |
| "clip_ratio": 0.0005630630766972899, |
| "completion_length": 121.21875, |
| "epoch": 0.11865101037684325, |
| "grad_norm": 0.024038543924689293, |
| "kl": 0.0018689564458327368, |
| "learning_rate": 6.894736842105264e-07, |
| "loss": 0.0001, |
| "num_tokens": 2525896.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 869 |
| }, |
| { |
| "clip_ratio": 0.00028153153834864497, |
| "epoch": 0.11878754778809394, |
| "grad_norm": 0.023961666971445084, |
| "kl": 0.001825891291446169, |
| "learning_rate": 6.842105263157896e-07, |
| "loss": -0.0, |
| "step": 870 |
| }, |
| { |
| "clip_ratio": 0.00028153153834864497, |
| "epoch": 0.11892408519934462, |
| "grad_norm": 0.024005061015486717, |
| "kl": 0.0018299317471246468, |
| "learning_rate": 6.789473684210526e-07, |
| "loss": -0.0, |
| "step": 871 |
| }, |
| { |
| "clip_ratio": 0.0006981982151046395, |
| "epoch": 0.1190606226105953, |
| "grad_norm": 0.024153009057044983, |
| "kl": 0.0018959494791488396, |
| "learning_rate": 6.736842105263158e-07, |
| "loss": 0.0001, |
| "step": 872 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.9375, |
| "epoch": 0.11919716002184598, |
| "grad_norm": 0.020175298675894737, |
| "kl": 0.0021377066386776278, |
| "learning_rate": 6.684210526315791e-07, |
| "loss": 0.0, |
| "num_tokens": 2537458.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 873 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11933369743309667, |
| "grad_norm": 0.020295897498726845, |
| "kl": 0.002052186724540661, |
| "learning_rate": 6.631578947368422e-07, |
| "loss": 0.0001, |
| "step": 874 |
| }, |
| { |
| "clip_ratio": 0.0001707650226308033, |
| "epoch": 0.11947023484434735, |
| "grad_norm": 0.02031334675848484, |
| "kl": 0.0021257563785184175, |
| "learning_rate": 6.578947368421053e-07, |
| "loss": 0.0001, |
| "step": 875 |
| }, |
| { |
| "clip_ratio": 0.000234962411923334, |
| "epoch": 0.11960677225559803, |
| "grad_norm": 0.020466400310397148, |
| "kl": 0.002226112668722635, |
| "learning_rate": 6.526315789473684e-07, |
| "loss": 0.0001, |
| "step": 876 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 124.3125, |
| "epoch": 0.11974330966684872, |
| "grad_norm": 0.017409300431609154, |
| "kl": 0.002158536493880092, |
| "learning_rate": 6.473684210526317e-07, |
| "loss": 0.0001, |
| "num_tokens": 2548268.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 877 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1198798470780994, |
| "grad_norm": 0.017360569909214973, |
| "kl": 0.002025924322879291, |
| "learning_rate": 6.421052631578948e-07, |
| "loss": 0.0001, |
| "step": 878 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12001638448935008, |
| "grad_norm": 0.017513062804937363, |
| "kl": 0.0020348432663013227, |
| "learning_rate": 6.368421052631579e-07, |
| "loss": 0.0001, |
| "step": 879 |
| }, |
| { |
| "clip_ratio": 0.00025201612152159214, |
| "epoch": 0.12015292190060077, |
| "grad_norm": 0.01726604998111725, |
| "kl": 0.0020535808343993267, |
| "learning_rate": 6.315789473684211e-07, |
| "loss": 0.0001, |
| "step": 880 |
| }, |
| { |
| "clip_ratio": 0.0006926970963831991, |
| "completion_length": 142.28125, |
| "epoch": 0.12028945931185145, |
| "grad_norm": 0.036571867763996124, |
| "kl": 0.002262445339511032, |
| "learning_rate": 6.263157894736844e-07, |
| "loss": 0.0001, |
| "num_tokens": 2560229.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 881 |
| }, |
| { |
| "clip_ratio": 0.0008497322705807164, |
| "epoch": 0.12042599672310213, |
| "grad_norm": 0.037133604288101196, |
| "kl": 0.0022962079056014773, |
| "learning_rate": 6.210526315789474e-07, |
| "loss": 0.0002, |
| "step": 882 |
| }, |
| { |
| "clip_ratio": 0.0005721866909880191, |
| "epoch": 0.12056253413435282, |
| "grad_norm": 0.03698795288801193, |
| "kl": 0.002319942112080753, |
| "learning_rate": 6.157894736842106e-07, |
| "loss": 0.0001, |
| "step": 883 |
| }, |
| { |
| "clip_ratio": 0.00020032051543239504, |
| "epoch": 0.1206990715456035, |
| "grad_norm": 0.036612141877412796, |
| "kl": 0.0023310945653065573, |
| "learning_rate": 6.105263157894738e-07, |
| "loss": 0.0001, |
| "step": 884 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.5625, |
| "epoch": 0.12083560895685418, |
| "grad_norm": 0.028590509667992592, |
| "kl": 0.0021473601445904933, |
| "learning_rate": 6.052631578947369e-07, |
| "loss": -0.0001, |
| "num_tokens": 2571031.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 885 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12097214636810485, |
| "grad_norm": 0.028788069263100624, |
| "kl": 0.002038355499280442, |
| "learning_rate": 6.000000000000001e-07, |
| "loss": -0.0, |
| "step": 886 |
| }, |
| { |
| "clip_ratio": 0.00035511364694684744, |
| "epoch": 0.12110868377935555, |
| "grad_norm": 0.02862849459052086, |
| "kl": 0.001932536375534255, |
| "learning_rate": 5.947368421052632e-07, |
| "loss": -0.0002, |
| "step": 887 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12124522119060623, |
| "grad_norm": 0.02891075611114502, |
| "kl": 0.0018507974418753292, |
| "learning_rate": 5.894736842105263e-07, |
| "loss": -0.0001, |
| "step": 888 |
| }, |
| { |
| "clip_ratio": 0.0002540650311857462, |
| "completion_length": 117.28125, |
| "epoch": 0.1213817586018569, |
| "grad_norm": 0.019024843350052834, |
| "kl": 0.002249214661787846, |
| "learning_rate": 5.842105263157896e-07, |
| "loss": -0.0001, |
| "num_tokens": 2582120.0, |
| "reward": 0.6031249761581421, |
| "reward_std": 0.07120190560817719, |
| "rewards/reward_fn": 0.6031249761581421, |
| "step": 889 |
| }, |
| { |
| "clip_ratio": 0.00021404109429568052, |
| "epoch": 0.1215182960131076, |
| "grad_norm": 0.019089598208665848, |
| "kl": 0.0020849486736551626, |
| "learning_rate": 5.789473684210526e-07, |
| "loss": -0.0002, |
| "step": 890 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12165483342435827, |
| "grad_norm": 0.018902519717812538, |
| "kl": 0.0022411892423406243, |
| "learning_rate": 5.736842105263158e-07, |
| "loss": -0.0001, |
| "step": 891 |
| }, |
| { |
| "clip_ratio": 0.0002540650311857462, |
| "epoch": 0.12179137083560895, |
| "grad_norm": 0.018905550241470337, |
| "kl": 0.002317437955753121, |
| "learning_rate": 5.68421052631579e-07, |
| "loss": -0.0, |
| "step": 892 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.96875, |
| "epoch": 0.12192790824685965, |
| "grad_norm": 0.035427920520305634, |
| "kl": 0.0019287737968625152, |
| "learning_rate": 5.631578947368421e-07, |
| "loss": -0.0, |
| "num_tokens": 2592995.0, |
| "reward": 0.32499998807907104, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.32499998807907104, |
| "step": 893 |
| }, |
| { |
| "clip_ratio": 0.0008413766045123339, |
| "epoch": 0.12206444565811032, |
| "grad_norm": 0.03540027514100075, |
| "kl": 0.0019906538364011794, |
| "learning_rate": 5.578947368421053e-07, |
| "loss": -0.0002, |
| "step": 894 |
| }, |
| { |
| "clip_ratio": 0.0010899427579715848, |
| "epoch": 0.122200983069361, |
| "grad_norm": 0.03542100265622139, |
| "kl": 0.002009846612054389, |
| "learning_rate": 5.526315789473684e-07, |
| "loss": 0.0001, |
| "step": 895 |
| }, |
| { |
| "clip_ratio": 0.0004976481723133475, |
| "epoch": 0.1223375204806117, |
| "grad_norm": 0.03547709807753563, |
| "kl": 0.0020012245713587618, |
| "learning_rate": 5.473684210526316e-07, |
| "loss": -0.0, |
| "step": 896 |
| }, |
| { |
| "clip_ratio": 0.0003513729461701587, |
| "completion_length": 144.25, |
| "epoch": 0.12247405789186237, |
| "grad_norm": 0.0332733653485775, |
| "kl": 0.002094986011798028, |
| "learning_rate": 5.421052631578948e-07, |
| "loss": 0.0, |
| "num_tokens": 2604815.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.25110572576522827, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 897 |
| }, |
| { |
| "clip_ratio": 0.0010349433432566002, |
| "epoch": 0.12261059530311305, |
| "grad_norm": 0.03256555646657944, |
| "kl": 0.002050184742984129, |
| "learning_rate": 5.368421052631579e-07, |
| "loss": 0.0002, |
| "step": 898 |
| }, |
| { |
| "clip_ratio": 0.0005613642715616152, |
| "epoch": 0.12274713271436373, |
| "grad_norm": 0.033078376203775406, |
| "kl": 0.0021014586727687856, |
| "learning_rate": 5.315789473684211e-07, |
| "loss": 0.0001, |
| "step": 899 |
| }, |
| { |
| "clip_ratio": 0.00041538292134646326, |
| "epoch": 0.12288367012561442, |
| "grad_norm": 0.03323504701256752, |
| "kl": 0.0020402646969159832, |
| "learning_rate": 5.263157894736843e-07, |
| "loss": -0.0001, |
| "step": 900 |
| }, |
| { |
| "clip_ratio": 0.0003972457634517923, |
| "completion_length": 131.0, |
| "epoch": 0.1230202075368651, |
| "grad_norm": 0.024151014164090157, |
| "kl": 0.0020411131608852884, |
| "learning_rate": 5.210526315789474e-07, |
| "loss": -0.0001, |
| "num_tokens": 2616651.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 901 |
| }, |
| { |
| "clip_ratio": 0.00026483050896786153, |
| "epoch": 0.12315674494811578, |
| "grad_norm": 0.024675916880369186, |
| "kl": 0.0021187742222537054, |
| "learning_rate": 5.157894736842106e-07, |
| "loss": -0.0001, |
| "step": 902 |
| }, |
| { |
| "clip_ratio": 0.0009654743334976956, |
| "epoch": 0.12329328235936647, |
| "grad_norm": 0.024687334895133972, |
| "kl": 0.0020260024430172052, |
| "learning_rate": 5.105263157894738e-07, |
| "loss": -0.0001, |
| "step": 903 |
| }, |
| { |
| "clip_ratio": 0.0009654743334976956, |
| "epoch": 0.12342981977061715, |
| "grad_norm": 0.023990610614418983, |
| "kl": 0.001970850536963553, |
| "learning_rate": 5.052631578947369e-07, |
| "loss": -0.0, |
| "step": 904 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 131.59375, |
| "epoch": 0.12356635718186783, |
| "grad_norm": 0.02948358654975891, |
| "kl": 0.0017617333833186422, |
| "learning_rate": 5.000000000000001e-07, |
| "loss": -0.0001, |
| "num_tokens": 2627562.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 905 |
| }, |
| { |
| "clip_ratio": 0.0002500000118743628, |
| "epoch": 0.12370289459311852, |
| "grad_norm": 0.029498081654310226, |
| "kl": 0.0018035039292954025, |
| "learning_rate": 4.947368421052632e-07, |
| "loss": -0.0001, |
| "step": 906 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1238394320043692, |
| "grad_norm": 0.029065372422337532, |
| "kl": 0.001829242224630434, |
| "learning_rate": 4.894736842105263e-07, |
| "loss": -0.0001, |
| "step": 907 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12397596941561988, |
| "grad_norm": 0.02955593168735504, |
| "kl": 0.001747171319948393, |
| "learning_rate": 4.842105263157895e-07, |
| "loss": 0.0, |
| "step": 908 |
| }, |
| { |
| "clip_ratio": 0.000662284146528691, |
| "completion_length": 118.5, |
| "epoch": 0.12411250682687057, |
| "grad_norm": 0.03342261537909508, |
| "kl": 0.0021508525096578524, |
| "learning_rate": 4.789473684210526e-07, |
| "loss": -0.0, |
| "num_tokens": 2638254.0, |
| "reward": 0.8031249642372131, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.8031249642372131, |
| "step": 909 |
| }, |
| { |
| "clip_ratio": 0.00034340660204179585, |
| "epoch": 0.12424904423812125, |
| "grad_norm": 0.03362087532877922, |
| "kl": 0.0020725269005197333, |
| "learning_rate": 4.7368421052631585e-07, |
| "loss": 0.0002, |
| "step": 910 |
| }, |
| { |
| "clip_ratio": 0.00034340660204179585, |
| "epoch": 0.12438558164937193, |
| "grad_norm": 0.03338223695755005, |
| "kl": 0.002021927133682766, |
| "learning_rate": 4.6842105263157896e-07, |
| "loss": 0.0001, |
| "step": 911 |
| }, |
| { |
| "clip_ratio": 0.00034340660204179585, |
| "epoch": 0.1245221190606226, |
| "grad_norm": 0.03381529077887535, |
| "kl": 0.002014444373344304, |
| "learning_rate": 4.631578947368422e-07, |
| "loss": -0.0, |
| "step": 912 |
| }, |
| { |
| "clip_ratio": 0.0005952381179668009, |
| "completion_length": 130.03125, |
| "epoch": 0.1246586564718733, |
| "grad_norm": 0.022914061322808266, |
| "kl": 0.0017131423273895052, |
| "learning_rate": 4.578947368421053e-07, |
| "loss": 0.0003, |
| "num_tokens": 2649651.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 913 |
| }, |
| { |
| "clip_ratio": 0.0006686990382149816, |
| "epoch": 0.12479519388312398, |
| "grad_norm": 0.023066677153110504, |
| "kl": 0.0018404278443995281, |
| "learning_rate": 4.526315789473685e-07, |
| "loss": 0.0004, |
| "step": 914 |
| }, |
| { |
| "clip_ratio": 0.00018168604583479464, |
| "epoch": 0.12493173129437465, |
| "grad_norm": 0.02291272021830082, |
| "kl": 0.001800586937861226, |
| "learning_rate": 4.473684210526316e-07, |
| "loss": 0.0003, |
| "step": 915 |
| }, |
| { |
| "clip_ratio": 0.00018168604583479464, |
| "epoch": 0.12506826870562535, |
| "grad_norm": 0.022816216573119164, |
| "kl": 0.0018408020005153958, |
| "learning_rate": 4.421052631578947e-07, |
| "loss": 0.0003, |
| "step": 916 |
| }, |
| { |
| "clip_ratio": 0.0004030029522255063, |
| "completion_length": 132.15625, |
| "epoch": 0.125204806116876, |
| "grad_norm": 0.024068795144557953, |
| "kl": 0.002098910128552234, |
| "learning_rate": 4.3684210526315794e-07, |
| "loss": -0.0001, |
| "num_tokens": 2661240.0, |
| "reward": 0.578125, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.578125, |
| "step": 917 |
| }, |
| { |
| "clip_ratio": 0.0004030029522255063, |
| "epoch": 0.1253413435281267, |
| "grad_norm": 0.023403236642479897, |
| "kl": 0.002160426525733783, |
| "learning_rate": 4.3157894736842105e-07, |
| "loss": 0.0001, |
| "step": 918 |
| }, |
| { |
| "clip_ratio": 0.0005795566248707473, |
| "epoch": 0.1254778809393774, |
| "grad_norm": 0.023773547261953354, |
| "kl": 0.0020677255215559853, |
| "learning_rate": 4.2631578947368427e-07, |
| "loss": 0.0, |
| "step": 919 |
| }, |
| { |
| "clip_ratio": 0.00017655367264524102, |
| "epoch": 0.12561441835062806, |
| "grad_norm": 0.02403651364147663, |
| "kl": 0.0021508349182113307, |
| "learning_rate": 4.210526315789474e-07, |
| "loss": 0.0001, |
| "step": 920 |
| }, |
| { |
| "clip_ratio": 0.00024801588733680546, |
| "completion_length": 128.65625, |
| "epoch": 0.12575095576187875, |
| "grad_norm": 0.02821282111108303, |
| "kl": 0.001836812205510796, |
| "learning_rate": 4.157894736842106e-07, |
| "loss": -0.0003, |
| "num_tokens": 2672441.0, |
| "reward": 0.6906249523162842, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.6906249523162842, |
| "step": 921 |
| }, |
| { |
| "clip_ratio": 0.00024801588733680546, |
| "epoch": 0.12588749317312944, |
| "grad_norm": 0.028158273547887802, |
| "kl": 0.0019186180161341326, |
| "learning_rate": 4.105263157894737e-07, |
| "loss": -0.0002, |
| "step": 922 |
| }, |
| { |
| "clip_ratio": 0.0007261889695655555, |
| "epoch": 0.1260240305843801, |
| "grad_norm": 0.028605299070477486, |
| "kl": 0.0018298066697752802, |
| "learning_rate": 4.052631578947368e-07, |
| "loss": 0.0001, |
| "step": 923 |
| }, |
| { |
| "clip_ratio": 0.0001516990305390209, |
| "epoch": 0.1261605679956308, |
| "grad_norm": 0.027679719030857086, |
| "kl": 0.001927206470099918, |
| "learning_rate": 4.0000000000000003e-07, |
| "loss": -0.0001, |
| "step": 924 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 138.96875, |
| "epoch": 0.1262971054068815, |
| "grad_norm": 0.013626189902424812, |
| "kl": 0.001803158293114393, |
| "learning_rate": 3.9473684210526315e-07, |
| "loss": 0.0, |
| "num_tokens": 2684516.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 925 |
| }, |
| { |
| "clip_ratio": 0.0001270325155928731, |
| "epoch": 0.12643364281813216, |
| "grad_norm": 0.013184582814574242, |
| "kl": 0.0018444102661305806, |
| "learning_rate": 3.8947368421052636e-07, |
| "loss": -0.0001, |
| "step": 926 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12657018022938285, |
| "grad_norm": 0.013402871787548065, |
| "kl": 0.0017522034195280867, |
| "learning_rate": 3.8421052631578947e-07, |
| "loss": -0.0, |
| "step": 927 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12670671764063354, |
| "grad_norm": 0.013282261788845062, |
| "kl": 0.0018042439696728252, |
| "learning_rate": 3.789473684210527e-07, |
| "loss": -0.0001, |
| "step": 928 |
| }, |
| { |
| "clip_ratio": 0.0005196314741624519, |
| "completion_length": 141.625, |
| "epoch": 0.1268432550518842, |
| "grad_norm": 0.034112170338630676, |
| "kl": 0.0018444620891386876, |
| "learning_rate": 3.736842105263158e-07, |
| "loss": -0.0001, |
| "num_tokens": 2696288.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 929 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1269797924631349, |
| "grad_norm": 0.034253526479005814, |
| "kl": 0.0017955962957785232, |
| "learning_rate": 3.6842105263157896e-07, |
| "loss": -0.0003, |
| "step": 930 |
| }, |
| { |
| "clip_ratio": 0.0003722258115885779, |
| "epoch": 0.1271163298743856, |
| "grad_norm": 0.03406032919883728, |
| "kl": 0.0016890436236280948, |
| "learning_rate": 3.6315789473684213e-07, |
| "loss": -0.0002, |
| "step": 931 |
| }, |
| { |
| "clip_ratio": 0.0006515669811051339, |
| "epoch": 0.12725286728563626, |
| "grad_norm": 0.0342014916241169, |
| "kl": 0.0017644154213485308, |
| "learning_rate": 3.578947368421053e-07, |
| "loss": -0.0001, |
| "step": 932 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 131.5625, |
| "epoch": 0.12738940469688695, |
| "grad_norm": 0.028642408549785614, |
| "kl": 0.0017243105503439438, |
| "learning_rate": 3.5263157894736846e-07, |
| "loss": -0.0001, |
| "num_tokens": 2707946.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 933 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12752594210813764, |
| "grad_norm": 0.02829623967409134, |
| "kl": 0.0017418616225768346, |
| "learning_rate": 3.4736842105263157e-07, |
| "loss": -0.0001, |
| "step": 934 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1276624795193883, |
| "grad_norm": 0.028566766530275345, |
| "kl": 0.0018181849009124562, |
| "learning_rate": 3.421052631578948e-07, |
| "loss": 0.0, |
| "step": 935 |
| }, |
| { |
| "clip_ratio": 0.00016447369125671685, |
| "epoch": 0.127799016930639, |
| "grad_norm": 0.028366414830088615, |
| "kl": 0.0016921994192671264, |
| "learning_rate": 3.368421052631579e-07, |
| "loss": -0.0001, |
| "step": 936 |
| }, |
| { |
| "clip_ratio": 0.0006108862871769816, |
| "completion_length": 155.28125, |
| "epoch": 0.1279355543418897, |
| "grad_norm": 0.0333460234105587, |
| "kl": 0.0017041650116880191, |
| "learning_rate": 3.315789473684211e-07, |
| "loss": -0.0003, |
| "num_tokens": 2720175.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 937 |
| }, |
| { |
| "clip_ratio": 0.00046177000331226736, |
| "epoch": 0.12807209175314035, |
| "grad_norm": 0.03320127725601196, |
| "kl": 0.0017484442232671427, |
| "learning_rate": 3.263157894736842e-07, |
| "loss": -0.0001, |
| "step": 938 |
| }, |
| { |
| "clip_ratio": 0.00014140272105578333, |
| "epoch": 0.12820862916439105, |
| "grad_norm": 0.03337475657463074, |
| "kl": 0.0017337162571493536, |
| "learning_rate": 3.210526315789474e-07, |
| "loss": -0.0001, |
| "step": 939 |
| }, |
| { |
| "clip_ratio": 0.0008734305156394839, |
| "epoch": 0.12834516657564174, |
| "grad_norm": 0.03332708403468132, |
| "kl": 0.001759997263434343, |
| "learning_rate": 3.1578947368421055e-07, |
| "loss": -0.0, |
| "step": 940 |
| }, |
| { |
| "clip_ratio": 0.0003595184243749827, |
| "completion_length": 128.875, |
| "epoch": 0.1284817039868924, |
| "grad_norm": 0.04191797599196434, |
| "kl": 0.0022904958823346533, |
| "learning_rate": 3.105263157894737e-07, |
| "loss": -0.0002, |
| "num_tokens": 2731411.0, |
| "reward": 0.71875, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.71875, |
| "step": 941 |
| }, |
| { |
| "clip_ratio": 0.0005447269359137863, |
| "epoch": 0.1286182413981431, |
| "grad_norm": 0.04139786958694458, |
| "kl": 0.002369653229834512, |
| "learning_rate": 3.052631578947369e-07, |
| "loss": -0.0, |
| "step": 942 |
| }, |
| { |
| "clip_ratio": 0.0006348488095682114, |
| "epoch": 0.12875477880939376, |
| "grad_norm": 0.041293784976005554, |
| "kl": 0.002344006352359429, |
| "learning_rate": 3.0000000000000004e-07, |
| "loss": -0.0002, |
| "step": 943 |
| }, |
| { |
| "clip_ratio": 0.00013469827536027879, |
| "epoch": 0.12889131622064445, |
| "grad_norm": 0.041487667709589005, |
| "kl": 0.0024770467080088565, |
| "learning_rate": 2.9473684210526315e-07, |
| "loss": -0.0001, |
| "step": 944 |
| }, |
| { |
| "clip_ratio": 0.0001594387722434476, |
| "completion_length": 123.25, |
| "epoch": 0.12902785363189515, |
| "grad_norm": 0.031169265508651733, |
| "kl": 0.0020621584189939313, |
| "learning_rate": 2.894736842105263e-07, |
| "loss": -0.0004, |
| "num_tokens": 2742219.0, |
| "reward": 0.4906250238418579, |
| "reward_std": 0.18370190262794495, |
| "rewards/reward_fn": 0.4906250238418579, |
| "step": 945 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1291643910431458, |
| "grad_norm": 0.03082355111837387, |
| "kl": 0.0021173268905840814, |
| "learning_rate": 2.842105263157895e-07, |
| "loss": -0.0002, |
| "step": 946 |
| }, |
| { |
| "clip_ratio": 0.0001594387722434476, |
| "epoch": 0.1293009284543965, |
| "grad_norm": 0.03010324388742447, |
| "kl": 0.0021255461579130497, |
| "learning_rate": 2.7894736842105264e-07, |
| "loss": -0.0001, |
| "step": 947 |
| }, |
| { |
| "clip_ratio": 0.0001460280327592045, |
| "epoch": 0.1294374658656472, |
| "grad_norm": 0.031324584037065506, |
| "kl": 0.001988987545701093, |
| "learning_rate": 2.736842105263158e-07, |
| "loss": -0.0003, |
| "step": 948 |
| }, |
| { |
| "clip_ratio": 0.000619448401266709, |
| "completion_length": 152.0, |
| "epoch": 0.12957400327689786, |
| "grad_norm": 0.02701011672616005, |
| "kl": 0.0018674684397410601, |
| "learning_rate": 2.6842105263157897e-07, |
| "loss": 0.0, |
| "num_tokens": 2754427.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 949 |
| }, |
| { |
| "clip_ratio": 0.00023086981673259288, |
| "epoch": 0.12971054068814855, |
| "grad_norm": 0.027033735066652298, |
| "kl": 0.0018265853186676395, |
| "learning_rate": 2.6315789473684213e-07, |
| "loss": 0.0001, |
| "step": 950 |
| }, |
| { |
| "clip_ratio": 0.00040395389805780724, |
| "epoch": 0.12984707809939924, |
| "grad_norm": 0.027212731540203094, |
| "kl": 0.0019295160582259996, |
| "learning_rate": 2.578947368421053e-07, |
| "loss": -0.0001, |
| "step": 951 |
| }, |
| { |
| "clip_ratio": 0.0006968785892240703, |
| "epoch": 0.1299836155106499, |
| "grad_norm": 0.02716096304357052, |
| "kl": 0.0018647334300112561, |
| "learning_rate": 2.5263157894736846e-07, |
| "loss": -0.0001, |
| "step": 952 |
| }, |
| { |
| "clip_ratio": 0.0007757260173093528, |
| "completion_length": 119.3125, |
| "epoch": 0.1301201529219006, |
| "grad_norm": 0.04574109613895416, |
| "kl": 0.0019425052614678862, |
| "learning_rate": 2.473684210526316e-07, |
| "loss": -0.0001, |
| "num_tokens": 2765617.0, |
| "reward": 0.6906249523162842, |
| "reward_std": 0.36360570788383484, |
| "rewards/reward_fn": 0.6906249523162842, |
| "step": 953 |
| }, |
| { |
| "clip_ratio": 0.000234962411923334, |
| "epoch": 0.1302566903331513, |
| "grad_norm": 0.04522733762860298, |
| "kl": 0.0018160702220484382, |
| "learning_rate": 2.4210526315789473e-07, |
| "loss": -0.0004, |
| "step": 954 |
| }, |
| { |
| "clip_ratio": 0.0008333333535119891, |
| "epoch": 0.13039322774440196, |
| "grad_norm": 0.04570286348462105, |
| "kl": 0.0019073822350037517, |
| "learning_rate": 2.3684210526315792e-07, |
| "loss": -0.0002, |
| "step": 955 |
| }, |
| { |
| "clip_ratio": 0.0006666666886303574, |
| "epoch": 0.13052976515565265, |
| "grad_norm": 0.04525072127580643, |
| "kl": 0.001965075010957662, |
| "learning_rate": 2.315789473684211e-07, |
| "loss": -0.0002, |
| "step": 956 |
| }, |
| { |
| "clip_ratio": 0.0005073317588539794, |
| "completion_length": 124.0, |
| "epoch": 0.13066630256690334, |
| "grad_norm": 0.03176407143473625, |
| "kl": 0.002387185357292765, |
| "learning_rate": 2.2631578947368425e-07, |
| "loss": 0.0, |
| "num_tokens": 2776405.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 957 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.130802839978154, |
| "grad_norm": 0.0319964699447155, |
| "kl": 0.0022256208067119587, |
| "learning_rate": 2.2105263157894736e-07, |
| "loss": -0.0002, |
| "step": 958 |
| }, |
| { |
| "clip_ratio": 0.00025826445198617876, |
| "epoch": 0.1309393773894047, |
| "grad_norm": 0.03232432156801224, |
| "kl": 0.0025238104517484317, |
| "learning_rate": 2.1578947368421053e-07, |
| "loss": 0.0001, |
| "step": 959 |
| }, |
| { |
| "clip_ratio": 0.0007217811944428831, |
| "epoch": 0.1310759148006554, |
| "grad_norm": 0.031727712601423264, |
| "kl": 0.0023681106831645593, |
| "learning_rate": 2.105263157894737e-07, |
| "loss": -0.0002, |
| "step": 960 |
| }, |
| { |
| "clip_ratio": 0.0003830051573459059, |
| "completion_length": 118.53125, |
| "epoch": 0.13121245221190606, |
| "grad_norm": 0.02657039277255535, |
| "kl": 0.002205699001933681, |
| "learning_rate": 2.0526315789473685e-07, |
| "loss": 0.0, |
| "num_tokens": 2787282.0, |
| "reward": 0.887499988079071, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.887499988079071, |
| "step": 961 |
| }, |
| { |
| "clip_ratio": 0.00016447369125671685, |
| "epoch": 0.13134898962315675, |
| "grad_norm": 0.026693396270275116, |
| "kl": 0.00212965470564086, |
| "learning_rate": 2.0000000000000002e-07, |
| "loss": 0.0, |
| "step": 962 |
| }, |
| { |
| "clip_ratio": 0.0004708462511189282, |
| "epoch": 0.13148552703440744, |
| "grad_norm": 0.026747936382889748, |
| "kl": 0.0022483005668618716, |
| "learning_rate": 1.9473684210526318e-07, |
| "loss": -0.0001, |
| "step": 963 |
| }, |
| { |
| "clip_ratio": 0.000635319942375645, |
| "epoch": 0.1316220644456581, |
| "grad_norm": 0.02631334215402603, |
| "kl": 0.002198568679887103, |
| "learning_rate": 1.8947368421052634e-07, |
| "loss": -0.0, |
| "step": 964 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.65625, |
| "epoch": 0.1317586018569088, |
| "grad_norm": 0.03489239513874054, |
| "kl": 0.0026326864990551258, |
| "learning_rate": 1.8421052631578948e-07, |
| "loss": -0.0001, |
| "num_tokens": 2798519.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 965 |
| }, |
| { |
| "clip_ratio": 0.0013032681017648429, |
| "epoch": 0.1318951392681595, |
| "grad_norm": 0.03495628386735916, |
| "kl": 0.0025636351347202435, |
| "learning_rate": 1.7894736842105265e-07, |
| "loss": -0.0003, |
| "step": 966 |
| }, |
| { |
| "clip_ratio": 0.0010461228230269626, |
| "epoch": 0.13203167667941015, |
| "grad_norm": 0.03498396277427673, |
| "kl": 0.002557776528192335, |
| "learning_rate": 1.7368421052631578e-07, |
| "loss": -0.0001, |
| "step": 967 |
| }, |
| { |
| "clip_ratio": 0.0011772600410040468, |
| "epoch": 0.13216821409066085, |
| "grad_norm": 0.035107169300317764, |
| "kl": 0.0026038188407255802, |
| "learning_rate": 1.6842105263157895e-07, |
| "loss": 0.0001, |
| "step": 968 |
| }, |
| { |
| "clip_ratio": 0.00022321428696159273, |
| "completion_length": 115.90625, |
| "epoch": 0.1323047515019115, |
| "grad_norm": 0.029477836564183235, |
| "kl": 0.002139107233233517, |
| "learning_rate": 1.631578947368421e-07, |
| "loss": -0.0001, |
| "num_tokens": 2809540.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 969 |
| }, |
| { |
| "clip_ratio": 0.0007052621658658609, |
| "epoch": 0.1324412889131622, |
| "grad_norm": 0.02851179987192154, |
| "kl": 0.0021293932059052167, |
| "learning_rate": 1.5789473684210527e-07, |
| "loss": -0.0, |
| "step": 970 |
| }, |
| { |
| "clip_ratio": 0.0002604166802484542, |
| "epoch": 0.1325778263244129, |
| "grad_norm": 0.02857079543173313, |
| "kl": 0.0020065300195710734, |
| "learning_rate": 1.5263157894736844e-07, |
| "loss": 0.0001, |
| "step": 971 |
| }, |
| { |
| "clip_ratio": 0.00044642857392318547, |
| "epoch": 0.13271436373566356, |
| "grad_norm": 0.029482480138540268, |
| "kl": 0.002193307056586491, |
| "learning_rate": 1.4736842105263158e-07, |
| "loss": 0.0002, |
| "step": 972 |
| }, |
| { |
| "clip_ratio": 0.000703705329215154, |
| "completion_length": 136.15625, |
| "epoch": 0.13285090114691425, |
| "grad_norm": 0.03487672656774521, |
| "kl": 0.0018428106286592083, |
| "learning_rate": 1.4210526315789474e-07, |
| "loss": 0.0001, |
| "num_tokens": 2820793.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 973 |
| }, |
| { |
| "clip_ratio": 0.000788291115895845, |
| "epoch": 0.13298743855816494, |
| "grad_norm": 0.0352611243724823, |
| "kl": 0.0017364407913191826, |
| "learning_rate": 1.368421052631579e-07, |
| "loss": 0.0, |
| "step": 974 |
| }, |
| { |
| "clip_ratio": 0.0010632339253788814, |
| "epoch": 0.1331239759694156, |
| "grad_norm": 0.03562775254249573, |
| "kl": 0.0017056907054211479, |
| "learning_rate": 1.3157894736842107e-07, |
| "loss": -0.0002, |
| "step": 975 |
| }, |
| { |
| "clip_ratio": 0.0002540650311857462, |
| "epoch": 0.1332605133806663, |
| "grad_norm": 0.03521440923213959, |
| "kl": 0.0017664205297478475, |
| "learning_rate": 1.2631578947368423e-07, |
| "loss": 0.0001, |
| "step": 976 |
| }, |
| { |
| "clip_ratio": 0.0006411774083971977, |
| "completion_length": 125.9375, |
| "epoch": 0.133397050791917, |
| "grad_norm": 0.04052438214421272, |
| "kl": 0.002075563157632132, |
| "learning_rate": 1.2105263157894737e-07, |
| "loss": 0.0001, |
| "num_tokens": 2831843.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 977 |
| }, |
| { |
| "clip_ratio": 0.0007323232421185821, |
| "epoch": 0.13353358820316766, |
| "grad_norm": 0.041622359305620193, |
| "kl": 0.0020363604762678733, |
| "learning_rate": 1.1578947368421054e-07, |
| "loss": 0.0002, |
| "step": 978 |
| }, |
| { |
| "clip_ratio": 0.00031565656536258757, |
| "epoch": 0.13367012561441835, |
| "grad_norm": 0.04189898818731308, |
| "kl": 0.0020175932950223796, |
| "learning_rate": 1.1052631578947368e-07, |
| "loss": 0.0001, |
| "step": 979 |
| }, |
| { |
| "clip_ratio": 0.0007421875197906047, |
| "epoch": 0.13380666302566904, |
| "grad_norm": 0.04237787052989006, |
| "kl": 0.001999835752940271, |
| "learning_rate": 1.0526315789473685e-07, |
| "loss": 0.0003, |
| "step": 980 |
| }, |
| { |
| "clip_ratio": 0.0004293356032576412, |
| "completion_length": 123.09375, |
| "epoch": 0.1339432004369197, |
| "grad_norm": 0.044372156262397766, |
| "kl": 0.0019276039001852041, |
| "learning_rate": 1.0000000000000001e-07, |
| "loss": 0.0, |
| "num_tokens": 2842758.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.35490381717681885, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 981 |
| }, |
| { |
| "clip_ratio": 0.000518681132234633, |
| "epoch": 0.1340797378481704, |
| "grad_norm": 0.04381715878844261, |
| "kl": 0.0021304004676494515, |
| "learning_rate": 9.473684210526317e-08, |
| "loss": -0.0002, |
| "step": 982 |
| }, |
| { |
| "clip_ratio": 0.00043168605770915747, |
| "epoch": 0.1342162752594211, |
| "grad_norm": 0.04373766854405403, |
| "kl": 0.001978013932784961, |
| "learning_rate": 8.947368421052632e-08, |
| "loss": -0.0001, |
| "step": 983 |
| }, |
| { |
| "clip_ratio": 0.0011094502406194806, |
| "epoch": 0.13435281267067176, |
| "grad_norm": 0.04434705153107643, |
| "kl": 0.0019842102301481646, |
| "learning_rate": 8.421052631578947e-08, |
| "loss": -0.0001, |
| "step": 984 |
| }, |
| { |
| "clip_ratio": 0.0008912696212064475, |
| "completion_length": 125.6875, |
| "epoch": 0.13448935008192245, |
| "grad_norm": 0.035390034317970276, |
| "kl": 0.0026560259429970756, |
| "learning_rate": 7.894736842105264e-08, |
| "loss": 0.0001, |
| "num_tokens": 2853588.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 985 |
| }, |
| { |
| "clip_ratio": 0.0007543278188677505, |
| "epoch": 0.13462588749317314, |
| "grad_norm": 0.0348115973174572, |
| "kl": 0.0027009638433810323, |
| "learning_rate": 7.368421052631579e-08, |
| "loss": 0.0002, |
| "step": 986 |
| }, |
| { |
| "clip_ratio": 0.0007222114945761859, |
| "epoch": 0.1347624249044238, |
| "grad_norm": 0.03600397706031799, |
| "kl": 0.002620842893520603, |
| "learning_rate": 6.842105263157895e-08, |
| "loss": 0.0002, |
| "step": 987 |
| }, |
| { |
| "clip_ratio": 0.0007801339961588383, |
| "epoch": 0.1348989623156745, |
| "grad_norm": 0.034362636506557465, |
| "kl": 0.0025556167329341406, |
| "learning_rate": 6.315789473684211e-08, |
| "loss": 0.0002, |
| "step": 988 |
| }, |
| { |
| "clip_ratio": 0.0004814189160242677, |
| "completion_length": 133.28125, |
| "epoch": 0.1350354997269252, |
| "grad_norm": 0.020515473559498787, |
| "kl": 0.001966405860002851, |
| "learning_rate": 5.789473684210527e-08, |
| "loss": 0.0001, |
| "num_tokens": 2865025.0, |
| "reward": 0.6343750357627869, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.6343750357627869, |
| "step": 989 |
| }, |
| { |
| "clip_ratio": 0.0006052327662473544, |
| "epoch": 0.13517203713817585, |
| "grad_norm": 0.020682375878095627, |
| "kl": 0.001828348807975999, |
| "learning_rate": 5.263157894736842e-08, |
| "loss": -0.0, |
| "step": 990 |
| }, |
| { |
| "clip_ratio": 0.00054094273946248, |
| "epoch": 0.13530857454942655, |
| "grad_norm": 0.020333580672740936, |
| "kl": 0.0018641406913957326, |
| "learning_rate": 4.7368421052631586e-08, |
| "loss": -0.0001, |
| "step": 991 |
| }, |
| { |
| "clip_ratio": 0.0003124999930150807, |
| "epoch": 0.13544511196067724, |
| "grad_norm": 0.020512109622359276, |
| "kl": 0.0017612487336009508, |
| "learning_rate": 4.2105263157894737e-08, |
| "loss": -0.0001, |
| "step": 992 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 123.15625, |
| "epoch": 0.1355816493719279, |
| "grad_norm": 0.021672414615750313, |
| "kl": 0.001844227123001474, |
| "learning_rate": 3.6842105263157894e-08, |
| "loss": 0.0002, |
| "num_tokens": 2876602.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 993 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1357181867831786, |
| "grad_norm": 0.02169492281973362, |
| "kl": 0.001965407580428291, |
| "learning_rate": 3.157894736842106e-08, |
| "loss": 0.0001, |
| "step": 994 |
| }, |
| { |
| "clip_ratio": 0.0002460629912093282, |
| "epoch": 0.13585472419442926, |
| "grad_norm": 0.021661533042788506, |
| "kl": 0.002062978146568639, |
| "learning_rate": 2.631578947368421e-08, |
| "loss": 0.0001, |
| "step": 995 |
| }, |
| { |
| "clip_ratio": 0.0004692772781709209, |
| "epoch": 0.13599126160567995, |
| "grad_norm": 0.021499216556549072, |
| "kl": 0.0019374883922864683, |
| "learning_rate": 2.1052631578947368e-08, |
| "loss": 0.0001, |
| "step": 996 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 122.59375, |
| "epoch": 0.13612779901693065, |
| "grad_norm": 0.026189453899860382, |
| "kl": 0.0020121177003602497, |
| "learning_rate": 1.578947368421053e-08, |
| "loss": 0.0, |
| "num_tokens": 2887857.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 997 |
| }, |
| { |
| "clip_ratio": 0.00010629251482896507, |
| "epoch": 0.1362643364281813, |
| "grad_norm": 0.02555716410279274, |
| "kl": 0.0019450822092039743, |
| "learning_rate": 1.0526315789473684e-08, |
| "loss": 0.0, |
| "step": 998 |
| }, |
| { |
| "clip_ratio": 0.00043367347097955644, |
| "epoch": 0.136400873839432, |
| "grad_norm": 0.025302477180957794, |
| "kl": 0.0021157398405193817, |
| "learning_rate": 5.263157894736842e-09, |
| "loss": -0.0002, |
| "step": 999 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1365374112506827, |
| "grad_norm": 0.025643914937973022, |
| "kl": 0.0020015074787806952, |
| "learning_rate": 0.0, |
| "loss": -0.0, |
| "step": 1000 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 1000, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 250, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 32, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
