| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.1365374112506827, |
| "eval_steps": 500, |
| "global_step": 1000, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.0625, |
| "epoch": 0.00013653741125068268, |
| "grad_norm": 0.08739373087882996, |
| "kl": 0.0006317789470813295, |
| "learning_rate": 1.0000000000000001e-07, |
| "loss": -0.0002, |
| "num_tokens": 11046.0, |
| "reward": 0.7437499761581421, |
| "reward_std": 0.19240379333496094, |
| "rewards/reward_fn": 0.7437499761581421, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.00027307482250136535, |
| "grad_norm": 0.08738827705383301, |
| "kl": 0.0006317789470813295, |
| "learning_rate": 2.0000000000000002e-07, |
| "loss": -0.0002, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.00040961223375204805, |
| "grad_norm": 0.08647612482309341, |
| "kl": 0.0006050676793165621, |
| "learning_rate": 3.0000000000000004e-07, |
| "loss": -0.0001, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0005461496450027307, |
| "grad_norm": 0.08993478119373322, |
| "kl": 0.0005967747410977609, |
| "learning_rate": 4.0000000000000003e-07, |
| "loss": -0.0, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 0.0009092581894947216, |
| "completion_length": 123.28125, |
| "epoch": 0.0006826870562534135, |
| "grad_norm": 0.09589574486017227, |
| "kl": 0.0005814487990392081, |
| "learning_rate": 5.000000000000001e-07, |
| "loss": 0.0, |
| "num_tokens": 22015.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.00021701389050576836, |
| "epoch": 0.0008192244675040961, |
| "grad_norm": 0.09353803843259811, |
| "kl": 0.0005364549010664632, |
| "learning_rate": 6.000000000000001e-07, |
| "loss": 0.0001, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.001004464298603125, |
| "epoch": 0.0009557618787547789, |
| "grad_norm": 0.10129779577255249, |
| "kl": 0.000562284074476338, |
| "learning_rate": 7.000000000000001e-07, |
| "loss": 0.0003, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.00021701389050576836, |
| "epoch": 0.0010922992900054614, |
| "grad_norm": 0.09212316572666168, |
| "kl": 0.0005370138956095616, |
| "learning_rate": 8.000000000000001e-07, |
| "loss": 0.0001, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.0009476926934439689, |
| "completion_length": 152.6875, |
| "epoch": 0.0012288367012561442, |
| "grad_norm": 0.10085771977901459, |
| "kl": 0.0005878389974895981, |
| "learning_rate": 9.000000000000001e-07, |
| "loss": -0.0, |
| "num_tokens": 34213.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.00126093348080758, |
| "epoch": 0.001365374112506827, |
| "grad_norm": 0.1008201465010643, |
| "kl": 0.0005425554691100842, |
| "learning_rate": 1.0000000000000002e-06, |
| "loss": 0.0001, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.0007717385306023061, |
| "epoch": 0.0015019115237575095, |
| "grad_norm": 0.10120692104101181, |
| "kl": 0.0005051454013482726, |
| "learning_rate": 1.1e-06, |
| "loss": 0.0, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.0012008554913336411, |
| "epoch": 0.0016384489350081922, |
| "grad_norm": 0.10010115802288055, |
| "kl": 0.0005464185369419283, |
| "learning_rate": 1.2000000000000002e-06, |
| "loss": -0.0001, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.0009030445798998699, |
| "completion_length": 145.59375, |
| "epoch": 0.001774986346258875, |
| "grad_norm": 0.09043823927640915, |
| "kl": 0.0005211641523601429, |
| "learning_rate": 1.3e-06, |
| "loss": 0.0002, |
| "num_tokens": 45996.0, |
| "reward": 0.6312500238418579, |
| "reward_std": 0.2836301922798157, |
| "rewards/reward_fn": 0.6312500238418579, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.0006189533451106399, |
| "epoch": 0.0019115237575095577, |
| "grad_norm": 0.09039849042892456, |
| "kl": 0.0005784431677966495, |
| "learning_rate": 1.4000000000000001e-06, |
| "loss": 0.0002, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.00031675209174863994, |
| "epoch": 0.0020480611687602405, |
| "grad_norm": 0.09077497571706772, |
| "kl": 0.0005650495991176285, |
| "learning_rate": 1.5e-06, |
| "loss": 0.0002, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.000734518383978866, |
| "epoch": 0.002184598580010923, |
| "grad_norm": 0.09000656008720398, |
| "kl": 0.0005918515398661839, |
| "learning_rate": 1.6000000000000001e-06, |
| "loss": 0.0001, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.0001270325155928731, |
| "completion_length": 151.8125, |
| "epoch": 0.0023211359912616056, |
| "grad_norm": 0.08065712451934814, |
| "kl": 0.0005675913225786644, |
| "learning_rate": 1.7000000000000002e-06, |
| "loss": 0.0001, |
| "num_tokens": 58366.0, |
| "reward": 0.578125, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.578125, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.00040856405394151807, |
| "epoch": 0.0024576734025122883, |
| "grad_norm": 0.08062373846769333, |
| "kl": 0.0006601297482120572, |
| "learning_rate": 1.8000000000000001e-06, |
| "loss": 0.0001, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.0001270325155928731, |
| "epoch": 0.002594210813762971, |
| "grad_norm": 0.08101943135261536, |
| "kl": 0.0006057311593394843, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": 0.0002, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.002730748225013654, |
| "grad_norm": 0.08095166832208633, |
| "kl": 0.0007306283941943548, |
| "learning_rate": 2.0000000000000003e-06, |
| "loss": -0.0001, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 175.4375, |
| "epoch": 0.0028672856362643366, |
| "grad_norm": 0.06835271418094635, |
| "kl": 0.0006434723027268774, |
| "learning_rate": 2.1000000000000002e-06, |
| "loss": 0.0, |
| "num_tokens": 71024.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.00040108305984176695, |
| "epoch": 0.003003823047515019, |
| "grad_norm": 0.06890249252319336, |
| "kl": 0.0007574095243398915, |
| "learning_rate": 2.2e-06, |
| "loss": -0.0001, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.0003356586676090956, |
| "epoch": 0.0031403604587657017, |
| "grad_norm": 0.07008787244558334, |
| "kl": 0.0008008444001461612, |
| "learning_rate": 2.3000000000000004e-06, |
| "loss": -0.0001, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.000147405662573874, |
| "epoch": 0.0032768978700163844, |
| "grad_norm": 0.06931491941213608, |
| "kl": 0.0008262821156677091, |
| "learning_rate": 2.4000000000000003e-06, |
| "loss": -0.0003, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.0011949757172260433, |
| "completion_length": 154.78125, |
| "epoch": 0.003413435281267067, |
| "grad_norm": 0.09464246779680252, |
| "kl": 0.0008019438510018517, |
| "learning_rate": 2.5e-06, |
| "loss": -0.0002, |
| "num_tokens": 82917.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.30735570192337036, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.0008628188807051629, |
| "epoch": 0.00354997269251775, |
| "grad_norm": 0.09641154855489731, |
| "kl": 0.0008660993607918499, |
| "learning_rate": 2.6e-06, |
| "loss": 0.0, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.000868194954819046, |
| "epoch": 0.0036865101037684327, |
| "grad_norm": 0.09419838339090347, |
| "kl": 0.0010259089904138818, |
| "learning_rate": 2.7000000000000004e-06, |
| "loss": -0.0001, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.0008564656018279493, |
| "epoch": 0.0038230475150191155, |
| "grad_norm": 0.08919408172369003, |
| "kl": 0.0012204149879835313, |
| "learning_rate": 2.8000000000000003e-06, |
| "loss": -0.0004, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.0008514263026881963, |
| "completion_length": 140.875, |
| "epoch": 0.003959584926269798, |
| "grad_norm": 0.08971544355154037, |
| "kl": 0.001275990204703703, |
| "learning_rate": 2.9e-06, |
| "loss": -0.0001, |
| "num_tokens": 94653.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.1948557198047638, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.004096122337520481, |
| "grad_norm": 0.08967753499746323, |
| "kl": 0.0016255443697446026, |
| "learning_rate": 3e-06, |
| "loss": -0.0, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.000954879040364176, |
| "epoch": 0.004232659748771164, |
| "grad_norm": 0.0948873832821846, |
| "kl": 0.0017126279599324334, |
| "learning_rate": 3.1000000000000004e-06, |
| "loss": -0.0001, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.0009331955807283521, |
| "epoch": 0.004369197160021846, |
| "grad_norm": 0.0915728509426117, |
| "kl": 0.0019823211387119954, |
| "learning_rate": 3.2000000000000003e-06, |
| "loss": -0.0001, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.00015624999650754035, |
| "completion_length": 156.3125, |
| "epoch": 0.004505734571272528, |
| "grad_norm": 0.07780904322862625, |
| "kl": 0.0020532281778287143, |
| "learning_rate": 3.3000000000000006e-06, |
| "loss": -0.0001, |
| "num_tokens": 107451.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.0005133637605467811, |
| "epoch": 0.004642271982523211, |
| "grad_norm": 0.07766616344451904, |
| "kl": 0.00238418992375955, |
| "learning_rate": 3.4000000000000005e-06, |
| "loss": 0.0, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.0011453818005975336, |
| "epoch": 0.004778809393773894, |
| "grad_norm": 0.0797438770532608, |
| "kl": 0.002831721540133003, |
| "learning_rate": 3.5e-06, |
| "loss": -0.0001, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.0010551480372669175, |
| "epoch": 0.004915346805024577, |
| "grad_norm": 0.07885287702083588, |
| "kl": 0.003094232171861222, |
| "learning_rate": 3.6000000000000003e-06, |
| "loss": -0.0001, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.0001382743357680738, |
| "completion_length": 159.625, |
| "epoch": 0.005051884216275259, |
| "grad_norm": 0.06207679212093353, |
| "kl": 0.0033535315305925906, |
| "learning_rate": 3.7e-06, |
| "loss": -0.0001, |
| "num_tokens": 119355.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.00040536836604587734, |
| "epoch": 0.005188421627525942, |
| "grad_norm": 0.062118254601955414, |
| "kl": 0.0036885815097775776, |
| "learning_rate": 3.8000000000000005e-06, |
| "loss": -0.0001, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.000545094400877133, |
| "epoch": 0.005324959038776625, |
| "grad_norm": 0.06358339637517929, |
| "kl": 0.0039351375453406945, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": 0.0, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.0005789794813608751, |
| "epoch": 0.005461496450027308, |
| "grad_norm": 0.06142238900065422, |
| "kl": 0.004162700774031691, |
| "learning_rate": 4.000000000000001e-06, |
| "loss": -0.0001, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.0004968734428985044, |
| "completion_length": 135.09375, |
| "epoch": 0.00559803386127799, |
| "grad_norm": 0.10866480320692062, |
| "kl": 0.00454872860427713, |
| "learning_rate": 4.1e-06, |
| "loss": 0.0002, |
| "num_tokens": 130746.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.0002297794126207009, |
| "epoch": 0.005734571272528673, |
| "grad_norm": 0.09940455108880997, |
| "kl": 0.004684018753323471, |
| "learning_rate": 4.2000000000000004e-06, |
| "loss": -0.0002, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.0002297794126207009, |
| "epoch": 0.005871108683779356, |
| "grad_norm": 0.09981972724199295, |
| "kl": 0.004767102662299294, |
| "learning_rate": 4.3e-06, |
| "loss": -0.0003, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.00021551724057644606, |
| "epoch": 0.006007646095030038, |
| "grad_norm": 0.0989937111735344, |
| "kl": 0.00463390517688822, |
| "learning_rate": 4.4e-06, |
| "loss": -0.0006, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 131.40625, |
| "epoch": 0.006144183506280721, |
| "grad_norm": 0.06329631805419922, |
| "kl": 0.004799115195055492, |
| "learning_rate": 4.5e-06, |
| "loss": -0.0, |
| "num_tokens": 142491.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.00017655367264524102, |
| "epoch": 0.006280720917531403, |
| "grad_norm": 0.06211785599589348, |
| "kl": 0.0047047465850482695, |
| "learning_rate": 4.600000000000001e-06, |
| "loss": -0.0002, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.006417258328782086, |
| "grad_norm": 0.060480426996946335, |
| "kl": 0.004680782469222322, |
| "learning_rate": 4.7e-06, |
| "loss": -0.0003, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.0008001700480235741, |
| "epoch": 0.006553795740032769, |
| "grad_norm": 0.06124988943338394, |
| "kl": 0.004546212963759899, |
| "learning_rate": 4.800000000000001e-06, |
| "loss": -0.0004, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 138.84375, |
| "epoch": 0.006690333151283452, |
| "grad_norm": 0.09592146426439285, |
| "kl": 0.004827649225262576, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": -0.0001, |
| "num_tokens": 153834.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.0011473706836113706, |
| "epoch": 0.006826870562534134, |
| "grad_norm": 0.09477215260267258, |
| "kl": 0.004913321821732097, |
| "learning_rate": 5e-06, |
| "loss": -0.0002, |
| "step": 50 |
| }, |
| { |
| "clip_ratio": 0.00017655367264524102, |
| "epoch": 0.006963407973784817, |
| "grad_norm": 0.09420502185821533, |
| "kl": 0.005133655802637804, |
| "learning_rate": 4.9947368421052636e-06, |
| "loss": -0.0006, |
| "step": 51 |
| }, |
| { |
| "clip_ratio": 0.0006438815908040851, |
| "epoch": 0.0070999453850355, |
| "grad_norm": 0.08837559819221497, |
| "kl": 0.005440949782496318, |
| "learning_rate": 4.989473684210527e-06, |
| "loss": -0.0011, |
| "step": 52 |
| }, |
| { |
| "clip_ratio": 0.0003052657120861113, |
| "completion_length": 140.1875, |
| "epoch": 0.007236482796286183, |
| "grad_norm": 0.07635519653558731, |
| "kl": 0.004967198314261623, |
| "learning_rate": 4.98421052631579e-06, |
| "loss": 0.0001, |
| "num_tokens": 165068.0, |
| "reward": 0.26875001192092896, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.26875001192092896, |
| "step": 53 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.007373020207536865, |
| "grad_norm": 0.07609064877033234, |
| "kl": 0.005105830910906661, |
| "learning_rate": 4.978947368421053e-06, |
| "loss": 0.0001, |
| "step": 54 |
| }, |
| { |
| "clip_ratio": 0.0004236875247443095, |
| "epoch": 0.007509557618787548, |
| "grad_norm": 0.07511620223522186, |
| "kl": 0.0052619903508457355, |
| "learning_rate": 4.973684210526316e-06, |
| "loss": -0.0003, |
| "step": 55 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.007646095030038231, |
| "grad_norm": 0.0717560425400734, |
| "kl": 0.005385495860537048, |
| "learning_rate": 4.968421052631579e-06, |
| "loss": -0.0006, |
| "step": 56 |
| }, |
| { |
| "clip_ratio": 0.0006011680379742756, |
| "completion_length": 143.28125, |
| "epoch": 0.007782632441288913, |
| "grad_norm": 0.10058116912841797, |
| "kl": 0.007372127176495269, |
| "learning_rate": 4.9631578947368426e-06, |
| "loss": 0.0004, |
| "num_tokens": 177057.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.30735570192337036, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 57 |
| }, |
| { |
| "clip_ratio": 0.0006176497845444828, |
| "epoch": 0.007919169852539596, |
| "grad_norm": 0.10059979557991028, |
| "kl": 0.007655073924979661, |
| "learning_rate": 4.957894736842106e-06, |
| "loss": -0.0, |
| "step": 58 |
| }, |
| { |
| "clip_ratio": 0.0002929344482254237, |
| "epoch": 0.008055707263790278, |
| "grad_norm": 0.0992376059293747, |
| "kl": 0.008138025958032813, |
| "learning_rate": 4.952631578947369e-06, |
| "loss": -0.0006, |
| "step": 59 |
| }, |
| { |
| "clip_ratio": 0.0005786725378129631, |
| "epoch": 0.008192244675040962, |
| "grad_norm": 0.09717098623514175, |
| "kl": 0.008423170140304137, |
| "learning_rate": 4.947368421052632e-06, |
| "loss": -0.0012, |
| "step": 60 |
| }, |
| { |
| "clip_ratio": 0.0005044226418249309, |
| "completion_length": 145.75, |
| "epoch": 0.008328782086291644, |
| "grad_norm": 0.07611903548240662, |
| "kl": 0.008245052813435905, |
| "learning_rate": 4.942105263157895e-06, |
| "loss": 0.0003, |
| "num_tokens": 189173.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 61 |
| }, |
| { |
| "clip_ratio": 0.0007626870938111097, |
| "epoch": 0.008465319497542327, |
| "grad_norm": 0.0758456438779831, |
| "kl": 0.009095820518268738, |
| "learning_rate": 4.936842105263158e-06, |
| "loss": -0.0001, |
| "step": 62 |
| }, |
| { |
| "clip_ratio": 0.0004581221583066508, |
| "epoch": 0.00860185690879301, |
| "grad_norm": 0.07229050993919373, |
| "kl": 0.009441651978704613, |
| "learning_rate": 4.9315789473684215e-06, |
| "loss": -0.0003, |
| "step": 63 |
| }, |
| { |
| "clip_ratio": 0.00047310019726864994, |
| "epoch": 0.008738394320043691, |
| "grad_norm": 0.07003813982009888, |
| "kl": 0.010120850245584734, |
| "learning_rate": 4.926315789473685e-06, |
| "loss": -0.0007, |
| "step": 64 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 121.375, |
| "epoch": 0.008874931731294375, |
| "grad_norm": 0.03567022085189819, |
| "kl": 0.011777133724535815, |
| "learning_rate": 4.921052631578948e-06, |
| "loss": 0.0001, |
| "num_tokens": 200529.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 65 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.009011469142545057, |
| "grad_norm": 0.03496097773313522, |
| "kl": 0.012535931979073212, |
| "learning_rate": 4.915789473684211e-06, |
| "loss": 0.0, |
| "step": 66 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.00914800655379574, |
| "grad_norm": 0.03556264564394951, |
| "kl": 0.012659623011131771, |
| "learning_rate": 4.910526315789474e-06, |
| "loss": -0.0, |
| "step": 67 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.009284543965046422, |
| "grad_norm": 0.03532101958990097, |
| "kl": 0.013175853382563218, |
| "learning_rate": 4.905263157894737e-06, |
| "loss": -0.0, |
| "step": 68 |
| }, |
| { |
| "clip_ratio": 0.0006605616363231093, |
| "completion_length": 124.90625, |
| "epoch": 0.009421081376297106, |
| "grad_norm": 0.05857838690280914, |
| "kl": 0.012000912392977625, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": 0.0003, |
| "num_tokens": 211414.0, |
| "reward": 0.8875000476837158, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.8875000476837158, |
| "step": 69 |
| }, |
| { |
| "clip_ratio": 0.0002765486715361476, |
| "epoch": 0.009557618787547788, |
| "grad_norm": 0.060380011796951294, |
| "kl": 0.012359667627606541, |
| "learning_rate": 4.894736842105264e-06, |
| "loss": 0.0001, |
| "step": 70 |
| }, |
| { |
| "clip_ratio": 0.0006605616363231093, |
| "epoch": 0.009694156198798471, |
| "grad_norm": 0.05945640802383423, |
| "kl": 0.012303199764573947, |
| "learning_rate": 4.889473684210527e-06, |
| "loss": -0.0, |
| "step": 71 |
| }, |
| { |
| "clip_ratio": 0.0006619231717195362, |
| "epoch": 0.009830693610049153, |
| "grad_norm": 0.052826281636953354, |
| "kl": 0.012528339851996861, |
| "learning_rate": 4.88421052631579e-06, |
| "loss": -0.0004, |
| "step": 72 |
| }, |
| { |
| "clip_ratio": 0.00036407963489182293, |
| "completion_length": 142.125, |
| "epoch": 0.009967231021299837, |
| "grad_norm": 0.08659490197896957, |
| "kl": 0.010383772038039751, |
| "learning_rate": 4.878947368421053e-06, |
| "loss": 0.0001, |
| "num_tokens": 223626.0, |
| "reward": 0.6031250357627869, |
| "reward_std": 0.23124998807907104, |
| "rewards/reward_fn": 0.6031250357627869, |
| "step": 73 |
| }, |
| { |
| "clip_ratio": 0.0002460629912093282, |
| "epoch": 0.010103768432550519, |
| "grad_norm": 0.08382167667150497, |
| "kl": 0.010327450014301576, |
| "learning_rate": 4.873684210526316e-06, |
| "loss": -0.0002, |
| "step": 74 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0102403058438012, |
| "grad_norm": 0.0809021145105362, |
| "kl": 0.010177173695410602, |
| "learning_rate": 4.8684210526315795e-06, |
| "loss": -0.0006, |
| "step": 75 |
| }, |
| { |
| "clip_ratio": 0.0005547723558265716, |
| "epoch": 0.010376843255051884, |
| "grad_norm": 0.07807288318872452, |
| "kl": 0.010375534344348125, |
| "learning_rate": 4.863157894736843e-06, |
| "loss": -0.0011, |
| "step": 76 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.15625, |
| "epoch": 0.010513380666302566, |
| "grad_norm": 0.09454550594091415, |
| "kl": 0.01368577299581375, |
| "learning_rate": 4.857894736842106e-06, |
| "loss": 0.0002, |
| "num_tokens": 235095.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 77 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.01064991807755325, |
| "grad_norm": 0.09300191700458527, |
| "kl": 0.013703606295166537, |
| "learning_rate": 4.852631578947369e-06, |
| "loss": -0.0002, |
| "step": 78 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.010786455488803932, |
| "grad_norm": 0.08737168461084366, |
| "kl": 0.01426290805102326, |
| "learning_rate": 4.847368421052631e-06, |
| "loss": -0.0006, |
| "step": 79 |
| }, |
| { |
| "clip_ratio": 0.00027173911803402007, |
| "epoch": 0.010922992900054615, |
| "grad_norm": 0.0839756429195404, |
| "kl": 0.014389160307473503, |
| "learning_rate": 4.842105263157895e-06, |
| "loss": -0.0011, |
| "step": 80 |
| }, |
| { |
| "clip_ratio": 0.00023148147738538682, |
| "completion_length": 115.40625, |
| "epoch": 0.011059530311305297, |
| "grad_norm": 0.0841306746006012, |
| "kl": 0.019477701251162216, |
| "learning_rate": 4.8368421052631585e-06, |
| "loss": 0.0003, |
| "num_tokens": 245552.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 81 |
| }, |
| { |
| "clip_ratio": 0.00021551724057644606, |
| "epoch": 0.01119606772255598, |
| "grad_norm": 0.08322307467460632, |
| "kl": 0.019782173665589653, |
| "learning_rate": 4.831578947368422e-06, |
| "loss": -0.0001, |
| "step": 82 |
| }, |
| { |
| "clip_ratio": 0.00021551724057644606, |
| "epoch": 0.011332605133806663, |
| "grad_norm": 0.08051823079586029, |
| "kl": 0.019718343231943436, |
| "learning_rate": 4.826315789473685e-06, |
| "loss": -0.0006, |
| "step": 83 |
| }, |
| { |
| "clip_ratio": 0.0007434187136823311, |
| "epoch": 0.011469142545057346, |
| "grad_norm": 0.07694952934980392, |
| "kl": 0.01945641718339175, |
| "learning_rate": 4.821052631578948e-06, |
| "loss": -0.0009, |
| "step": 84 |
| }, |
| { |
| "clip_ratio": 0.000730298925191164, |
| "completion_length": 113.375, |
| "epoch": 0.011605679956308028, |
| "grad_norm": 0.0934474989771843, |
| "kl": 0.01406104041961953, |
| "learning_rate": 4.815789473684211e-06, |
| "loss": 0.0001, |
| "num_tokens": 256096.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 85 |
| }, |
| { |
| "clip_ratio": 0.0006386408931575716, |
| "epoch": 0.011742217367558712, |
| "grad_norm": 0.09243673086166382, |
| "kl": 0.013716014829697087, |
| "learning_rate": 4.8105263157894735e-06, |
| "loss": -0.0002, |
| "step": 86 |
| }, |
| { |
| "clip_ratio": 0.00016983695968519896, |
| "epoch": 0.011878754778809394, |
| "grad_norm": 0.08725881576538086, |
| "kl": 0.013890811271267012, |
| "learning_rate": 4.8052631578947375e-06, |
| "loss": -0.0007, |
| "step": 87 |
| }, |
| { |
| "clip_ratio": 0.00024801588733680546, |
| "epoch": 0.012015292190060076, |
| "grad_norm": 0.07987057417631149, |
| "kl": 0.013736142864217982, |
| "learning_rate": 4.800000000000001e-06, |
| "loss": -0.0012, |
| "step": 88 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.03125, |
| "epoch": 0.01215182960131076, |
| "grad_norm": 0.035524703562259674, |
| "kl": 0.014395015765330754, |
| "learning_rate": 4.794736842105264e-06, |
| "loss": 0.0001, |
| "num_tokens": 267685.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 89 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.012288367012561441, |
| "grad_norm": 0.03528278321027756, |
| "kl": 0.014380123306182213, |
| "learning_rate": 4.789473684210527e-06, |
| "loss": 0.0001, |
| "step": 90 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.012424904423812125, |
| "grad_norm": 0.03564177826046944, |
| "kl": 0.014681547458167188, |
| "learning_rate": 4.78421052631579e-06, |
| "loss": -0.0, |
| "step": 91 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.012561441835062807, |
| "grad_norm": 0.03350580483675003, |
| "kl": 0.01473659290059004, |
| "learning_rate": 4.778947368421053e-06, |
| "loss": -0.0001, |
| "step": 92 |
| }, |
| { |
| "clip_ratio": 0.0010092409647768363, |
| "completion_length": 116.34375, |
| "epoch": 0.01269797924631349, |
| "grad_norm": 0.11783453822135925, |
| "kl": 0.016927717690123245, |
| "learning_rate": 4.773684210526316e-06, |
| "loss": 0.0002, |
| "num_tokens": 278480.0, |
| "reward": 0.46562501788139343, |
| "reward_std": 0.36360570788383484, |
| "rewards/reward_fn": 0.46562501788139343, |
| "step": 93 |
| }, |
| { |
| "clip_ratio": 0.0005676986766047776, |
| "epoch": 0.012834516657564172, |
| "grad_norm": 0.1174498200416565, |
| "kl": 0.016682636385667138, |
| "learning_rate": 4.76842105263158e-06, |
| "loss": -0.0001, |
| "step": 94 |
| }, |
| { |
| "clip_ratio": 0.00021258502965793014, |
| "epoch": 0.012971054068814856, |
| "grad_norm": 0.1170513778924942, |
| "kl": 0.016236918556387536, |
| "learning_rate": 4.763157894736842e-06, |
| "loss": -0.001, |
| "step": 95 |
| }, |
| { |
| "clip_ratio": 0.0008842054230626673, |
| "epoch": 0.013107591480065538, |
| "grad_norm": 0.11030783504247665, |
| "kl": 0.016805329025373794, |
| "learning_rate": 4.757894736842106e-06, |
| "loss": -0.0018, |
| "step": 96 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.34375, |
| "epoch": 0.013244128891316221, |
| "grad_norm": 0.05533390864729881, |
| "kl": 0.012452395327272825, |
| "learning_rate": 4.752631578947369e-06, |
| "loss": 0.0, |
| "num_tokens": 289523.0, |
| "reward": 0.859375, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.859375, |
| "step": 97 |
| }, |
| { |
| "clip_ratio": 0.00028935185400769114, |
| "epoch": 0.013380666302566903, |
| "grad_norm": 0.05383645370602608, |
| "kl": 0.012898564287752379, |
| "learning_rate": 4.747368421052632e-06, |
| "loss": -0.0, |
| "step": 98 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.013517203713817587, |
| "grad_norm": 0.045518334954977036, |
| "kl": 0.012721069382678252, |
| "learning_rate": 4.7421052631578954e-06, |
| "loss": -0.0002, |
| "step": 99 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.013653741125068269, |
| "grad_norm": 0.043640851974487305, |
| "kl": 0.013114535111526493, |
| "learning_rate": 4.736842105263158e-06, |
| "loss": -0.0004, |
| "step": 100 |
| }, |
| { |
| "clip_ratio": 0.0003751235635718331, |
| "completion_length": 133.21875, |
| "epoch": 0.01379027853631895, |
| "grad_norm": 0.08343350887298584, |
| "kl": 0.011969710292760283, |
| "learning_rate": 4.731578947368422e-06, |
| "loss": 0.0003, |
| "num_tokens": 301274.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 101 |
| }, |
| { |
| "clip_ratio": 0.0003751235635718331, |
| "epoch": 0.013926815947569634, |
| "grad_norm": 0.08278653025627136, |
| "kl": 0.011722918432496954, |
| "learning_rate": 4.726315789473684e-06, |
| "loss": -0.0001, |
| "step": 102 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.014063353358820316, |
| "grad_norm": 0.08366868644952774, |
| "kl": 0.011790596836362965, |
| "learning_rate": 4.721052631578948e-06, |
| "loss": -0.0006, |
| "step": 103 |
| }, |
| { |
| "clip_ratio": 0.00020161290012765676, |
| "epoch": 0.014199890770071, |
| "grad_norm": 0.07245514541864395, |
| "kl": 0.011914224931388162, |
| "learning_rate": 4.71578947368421e-06, |
| "loss": -0.001, |
| "step": 104 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.34375, |
| "epoch": 0.014336428181321682, |
| "grad_norm": 0.06334606558084488, |
| "kl": 0.014175733827869408, |
| "learning_rate": 4.710526315789474e-06, |
| "loss": 0.0002, |
| "num_tokens": 312005.0, |
| "reward": 0.9156249761581421, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.9156249761581421, |
| "step": 105 |
| }, |
| { |
| "clip_ratio": 0.00028153153834864497, |
| "epoch": 0.014472965592572365, |
| "grad_norm": 0.06191639602184296, |
| "kl": 0.014437062345677987, |
| "learning_rate": 4.705263157894738e-06, |
| "loss": 0.0, |
| "step": 106 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.014609503003823047, |
| "grad_norm": 0.06282703578472137, |
| "kl": 0.014868097176076844, |
| "learning_rate": 4.7e-06, |
| "loss": -0.0002, |
| "step": 107 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.01474604041507373, |
| "grad_norm": 0.06213715299963951, |
| "kl": 0.015245917427819222, |
| "learning_rate": 4.694736842105264e-06, |
| "loss": -0.0006, |
| "step": 108 |
| }, |
| { |
| "clip_ratio": 0.00048828125, |
| "completion_length": 114.46875, |
| "epoch": 0.014882577826324413, |
| "grad_norm": 0.06244153529405594, |
| "kl": 0.014229173626517877, |
| "learning_rate": 4.689473684210526e-06, |
| "loss": 0.0002, |
| "num_tokens": 323008.0, |
| "reward": 0.856249988079071, |
| "reward_std": 0.12358209490776062, |
| "rewards/reward_fn": 0.856249988079071, |
| "step": 109 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.015019115237575096, |
| "grad_norm": 0.06384127587080002, |
| "kl": 0.014452488074311987, |
| "learning_rate": 4.68421052631579e-06, |
| "loss": -0.0001, |
| "step": 110 |
| }, |
| { |
| "clip_ratio": 0.000732421875, |
| "epoch": 0.015155652648825778, |
| "grad_norm": 0.05795735865831375, |
| "kl": 0.014755003750906326, |
| "learning_rate": 4.6789473684210525e-06, |
| "loss": -0.0003, |
| "step": 111 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.015292190060076462, |
| "grad_norm": 0.06043582037091255, |
| "kl": 0.014582981850253418, |
| "learning_rate": 4.6736842105263166e-06, |
| "loss": -0.0004, |
| "step": 112 |
| }, |
| { |
| "clip_ratio": 0.0005965908931102604, |
| "completion_length": 109.96875, |
| "epoch": 0.015428727471327144, |
| "grad_norm": 0.06778660416603088, |
| "kl": 0.015787007549079135, |
| "learning_rate": 4.668421052631579e-06, |
| "loss": 0.0003, |
| "num_tokens": 333667.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 113 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.015565264882577826, |
| "grad_norm": 0.07079855352640152, |
| "kl": 0.015863822191022336, |
| "learning_rate": 4.663157894736842e-06, |
| "loss": -0.0, |
| "step": 114 |
| }, |
| { |
| "clip_ratio": 0.0005342270596884191, |
| "epoch": 0.015701802293828507, |
| "grad_norm": 0.07030012458562851, |
| "kl": 0.015859683539019898, |
| "learning_rate": 4.657894736842106e-06, |
| "loss": -0.0003, |
| "step": 115 |
| }, |
| { |
| "clip_ratio": 0.00032552084303461015, |
| "epoch": 0.015838339705079193, |
| "grad_norm": 0.05977308005094528, |
| "kl": 0.015181809227215126, |
| "learning_rate": 4.652631578947368e-06, |
| "loss": -0.0007, |
| "step": 116 |
| }, |
| { |
| "clip_ratio": 0.000294811325147748, |
| "completion_length": 102.75, |
| "epoch": 0.015974877116329875, |
| "grad_norm": 0.09741310030221939, |
| "kl": 0.016162807471118867, |
| "learning_rate": 4.647368421052632e-06, |
| "loss": 0.0004, |
| "num_tokens": 343995.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 117 |
| }, |
| { |
| "clip_ratio": 0.00020973154460079968, |
| "epoch": 0.016111414527580557, |
| "grad_norm": 0.09549403935670853, |
| "kl": 0.015964071644702926, |
| "learning_rate": 4.642105263157895e-06, |
| "loss": 0.0001, |
| "step": 118 |
| }, |
| { |
| "clip_ratio": 0.0010666992020560429, |
| "epoch": 0.01624795193883124, |
| "grad_norm": 0.09260634332895279, |
| "kl": 0.016316924724378623, |
| "learning_rate": 4.636842105263159e-06, |
| "loss": -0.0007, |
| "step": 119 |
| }, |
| { |
| "clip_ratio": 0.0005621563323074952, |
| "epoch": 0.016384489350081924, |
| "grad_norm": 0.07998410612344742, |
| "kl": 0.016282340366160497, |
| "learning_rate": 4.631578947368421e-06, |
| "loss": -0.0016, |
| "step": 120 |
| }, |
| { |
| "clip_ratio": 0.0005777268670499325, |
| "completion_length": 105.71875, |
| "epoch": 0.016521026761332606, |
| "grad_norm": 0.11327945441007614, |
| "kl": 0.014576794725144282, |
| "learning_rate": 4.626315789473684e-06, |
| "loss": -0.0, |
| "num_tokens": 354366.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 121 |
| }, |
| { |
| "clip_ratio": 0.0010138368816114962, |
| "epoch": 0.016657564172583288, |
| "grad_norm": 0.11017230898141861, |
| "kl": 0.015154679873376153, |
| "learning_rate": 4.621052631578948e-06, |
| "loss": -0.0003, |
| "step": 122 |
| }, |
| { |
| "clip_ratio": 0.0029109671886544675, |
| "epoch": 0.01679410158383397, |
| "grad_norm": 0.09320779144763947, |
| "kl": 0.01626152906101197, |
| "learning_rate": 4.6157894736842105e-06, |
| "loss": -0.001, |
| "step": 123 |
| }, |
| { |
| "clip_ratio": 0.003067635203478858, |
| "epoch": 0.016930638995084655, |
| "grad_norm": 0.08714654296636581, |
| "kl": 0.01717385504161939, |
| "learning_rate": 4.6105263157894745e-06, |
| "loss": -0.0019, |
| "step": 124 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.625, |
| "epoch": 0.017067176406335337, |
| "grad_norm": 0.06164494529366493, |
| "kl": 0.022742171786376275, |
| "learning_rate": 4.605263157894737e-06, |
| "loss": 0.0001, |
| "num_tokens": 365118.0, |
| "reward": 0.4937500059604645, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.4937500059604645, |
| "step": 125 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.01720371381758602, |
| "grad_norm": 0.05765366554260254, |
| "kl": 0.023318831241340376, |
| "learning_rate": 4.600000000000001e-06, |
| "loss": 0.0001, |
| "step": 126 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0173402512288367, |
| "grad_norm": 0.057430919259786606, |
| "kl": 0.024255198208265938, |
| "learning_rate": 4.594736842105263e-06, |
| "loss": -0.0001, |
| "step": 127 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.017476788640087382, |
| "grad_norm": 0.0521625392138958, |
| "kl": 0.024568809458287433, |
| "learning_rate": 4.589473684210526e-06, |
| "loss": -0.0003, |
| "step": 128 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.4375, |
| "epoch": 0.017613326051338068, |
| "grad_norm": 0.06752870976924896, |
| "kl": 0.02813084190711379, |
| "learning_rate": 4.5842105263157895e-06, |
| "loss": 0.0003, |
| "num_tokens": 375444.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 129 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.01774986346258875, |
| "grad_norm": 0.06426940858364105, |
| "kl": 0.02811072752228938, |
| "learning_rate": 4.578947368421053e-06, |
| "loss": -0.0001, |
| "step": 130 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.01788640087383943, |
| "grad_norm": 0.052824705839157104, |
| "kl": 0.02794748015003279, |
| "learning_rate": 4.573684210526317e-06, |
| "loss": -0.0003, |
| "step": 131 |
| }, |
| { |
| "clip_ratio": 0.0005663055635523051, |
| "epoch": 0.018022938285090113, |
| "grad_norm": 0.05740916728973389, |
| "kl": 0.027399677026551217, |
| "learning_rate": 4.568421052631579e-06, |
| "loss": -0.0006, |
| "step": 132 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 95.0625, |
| "epoch": 0.0181594756963408, |
| "grad_norm": 0.08138228207826614, |
| "kl": 0.021528464392758906, |
| "learning_rate": 4.563157894736843e-06, |
| "loss": 0.0004, |
| "num_tokens": 385342.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 133 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.01829601310759148, |
| "grad_norm": 0.07547755539417267, |
| "kl": 0.021013552715885453, |
| "learning_rate": 4.557894736842105e-06, |
| "loss": 0.0001, |
| "step": 134 |
| }, |
| { |
| "clip_ratio": 0.0004340277810115367, |
| "epoch": 0.018432550518842163, |
| "grad_norm": 0.06484001874923706, |
| "kl": 0.020921625837218016, |
| "learning_rate": 4.552631578947369e-06, |
| "loss": -0.0003, |
| "step": 135 |
| }, |
| { |
| "clip_ratio": 0.0004340277810115367, |
| "epoch": 0.018569087930092845, |
| "grad_norm": 0.051754601299762726, |
| "kl": 0.021819816451170482, |
| "learning_rate": 4.547368421052632e-06, |
| "loss": -0.0008, |
| "step": 136 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.28125, |
| "epoch": 0.01870562534134353, |
| "grad_norm": 0.04303466156125069, |
| "kl": 0.02289676680811681, |
| "learning_rate": 4.542105263157895e-06, |
| "loss": 0.0002, |
| "num_tokens": 396003.0, |
| "reward": 0.971875011920929, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.971875011920929, |
| "step": 137 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.018842162752594212, |
| "grad_norm": 0.04287838935852051, |
| "kl": 0.02256237791152671, |
| "learning_rate": 4.536842105263158e-06, |
| "loss": 0.0001, |
| "step": 138 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.018978700163844894, |
| "grad_norm": 0.04330320656299591, |
| "kl": 0.022393779450794682, |
| "learning_rate": 4.531578947368421e-06, |
| "loss": 0.0001, |
| "step": 139 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.019115237575095576, |
| "grad_norm": 0.039670608937740326, |
| "kl": 0.022108391451183707, |
| "learning_rate": 4.526315789473685e-06, |
| "loss": -0.0002, |
| "step": 140 |
| }, |
| { |
| "clip_ratio": 0.0011110177874797955, |
| "completion_length": 124.75, |
| "epoch": 0.019251774986346257, |
| "grad_norm": 0.08559431880712509, |
| "kl": 0.02040508067875635, |
| "learning_rate": 4.5210526315789475e-06, |
| "loss": 0.0004, |
| "num_tokens": 407619.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 141 |
| }, |
| { |
| "clip_ratio": 0.0009295805648434907, |
| "epoch": 0.019388312397596943, |
| "grad_norm": 0.08730953186750412, |
| "kl": 0.0202211801224621, |
| "learning_rate": 4.5157894736842115e-06, |
| "loss": 0.0001, |
| "step": 142 |
| }, |
| { |
| "clip_ratio": 0.0001990445889532566, |
| "epoch": 0.019524849808847625, |
| "grad_norm": 0.08218631148338318, |
| "kl": 0.020368822690215893, |
| "learning_rate": 4.510526315789474e-06, |
| "loss": -0.0005, |
| "step": 143 |
| }, |
| { |
| "clip_ratio": 0.0001990445889532566, |
| "epoch": 0.019661387220098307, |
| "grad_norm": 0.07692748308181763, |
| "kl": 0.02033531089546159, |
| "learning_rate": 4.505263157894737e-06, |
| "loss": -0.0011, |
| "step": 144 |
| }, |
| { |
| "clip_ratio": 0.00027173911803402007, |
| "completion_length": 93.125, |
| "epoch": 0.01979792463134899, |
| "grad_norm": 0.06828766316175461, |
| "kl": 0.026619675249094144, |
| "learning_rate": 4.5e-06, |
| "loss": 0.0003, |
| "num_tokens": 417195.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 145 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.019934462042599674, |
| "grad_norm": 0.07163087278604507, |
| "kl": 0.027274078922346234, |
| "learning_rate": 4.494736842105263e-06, |
| "loss": 0.0001, |
| "step": 146 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.020070999453850356, |
| "grad_norm": 0.0677010715007782, |
| "kl": 0.02618727515800856, |
| "learning_rate": 4.489473684210527e-06, |
| "loss": -0.0003, |
| "step": 147 |
| }, |
| { |
| "clip_ratio": 0.0005972599610686302, |
| "epoch": 0.020207536865101038, |
| "grad_norm": 0.05918688699603081, |
| "kl": 0.0264243567653466, |
| "learning_rate": 4.48421052631579e-06, |
| "loss": -0.0005, |
| "step": 148 |
| }, |
| { |
| "clip_ratio": 0.00023854961909819394, |
| "completion_length": 114.15625, |
| "epoch": 0.02034407427635172, |
| "grad_norm": 0.10522865504026413, |
| "kl": 0.025243330223020166, |
| "learning_rate": 4.478947368421054e-06, |
| "loss": 0.0004, |
| "num_tokens": 427584.0, |
| "reward": 0.6343750357627869, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.6343750357627869, |
| "step": 149 |
| }, |
| { |
| "clip_ratio": 0.00023854961909819394, |
| "epoch": 0.0204806116876024, |
| "grad_norm": 0.10407311469316483, |
| "kl": 0.02505102547002025, |
| "learning_rate": 4.473684210526316e-06, |
| "loss": -0.0003, |
| "step": 150 |
| }, |
| { |
| "clip_ratio": 0.0009035185066750273, |
| "epoch": 0.020617149098853087, |
| "grad_norm": 0.10190074890851974, |
| "kl": 0.025200023461366072, |
| "learning_rate": 4.468421052631579e-06, |
| "loss": -0.001, |
| "step": 151 |
| }, |
| { |
| "clip_ratio": 0.0014054116763873026, |
| "epoch": 0.02075368651010377, |
| "grad_norm": 0.0961143970489502, |
| "kl": 0.025511470768833533, |
| "learning_rate": 4.463157894736842e-06, |
| "loss": -0.0019, |
| "step": 152 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.71875, |
| "epoch": 0.02089022392135445, |
| "grad_norm": 0.059831857681274414, |
| "kl": 0.023090639937436208, |
| "learning_rate": 4.4578947368421054e-06, |
| "loss": 0.0003, |
| "num_tokens": 438211.0, |
| "reward": 0.3812500238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.3812500238418579, |
| "step": 153 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.021026761332605132, |
| "grad_norm": 0.059171441942453384, |
| "kl": 0.02331032755319029, |
| "learning_rate": 4.452631578947369e-06, |
| "loss": 0.0001, |
| "step": 154 |
| }, |
| { |
| "clip_ratio": 0.0006432863010559231, |
| "epoch": 0.021163298743855818, |
| "grad_norm": 0.05819160118699074, |
| "kl": 0.023676532495301217, |
| "learning_rate": 4.447368421052632e-06, |
| "loss": -0.0, |
| "step": 155 |
| }, |
| { |
| "clip_ratio": 0.0005681818001903594, |
| "epoch": 0.0212998361551065, |
| "grad_norm": 0.05214538797736168, |
| "kl": 0.02407641662284732, |
| "learning_rate": 4.442105263157896e-06, |
| "loss": -0.0004, |
| "step": 156 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.25, |
| "epoch": 0.02143637356635718, |
| "grad_norm": 0.06498148292303085, |
| "kl": 0.028692521911580116, |
| "learning_rate": 4.436842105263158e-06, |
| "loss": 0.0002, |
| "num_tokens": 448495.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 157 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.021572910977607863, |
| "grad_norm": 0.06544307619333267, |
| "kl": 0.029219213291071355, |
| "learning_rate": 4.431578947368421e-06, |
| "loss": 0.0003, |
| "step": 158 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.02170944838885855, |
| "grad_norm": 0.06307835131883621, |
| "kl": 0.0294132886629086, |
| "learning_rate": 4.426315789473684e-06, |
| "loss": -0.0002, |
| "step": 159 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.02184598580010923, |
| "grad_norm": 0.06254979223012924, |
| "kl": 0.03018665249692276, |
| "learning_rate": 4.4210526315789476e-06, |
| "loss": -0.0003, |
| "step": 160 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 122.75, |
| "epoch": 0.021982523211359913, |
| "grad_norm": 0.09002089500427246, |
| "kl": 0.025298897293396294, |
| "learning_rate": 4.415789473684211e-06, |
| "loss": 0.0001, |
| "num_tokens": 459767.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.2337018847465515, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 161 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.022119060622610594, |
| "grad_norm": 0.0899246335029602, |
| "kl": 0.025724691950017586, |
| "learning_rate": 4.410526315789474e-06, |
| "loss": -0.0, |
| "step": 162 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.022255598033861276, |
| "grad_norm": 0.08925420790910721, |
| "kl": 0.02707095700316131, |
| "learning_rate": 4.405263157894737e-06, |
| "loss": -0.0007, |
| "step": 163 |
| }, |
| { |
| "clip_ratio": 0.00017361111531499773, |
| "epoch": 0.02239213544511196, |
| "grad_norm": 0.0802987664937973, |
| "kl": 0.027386941597796977, |
| "learning_rate": 4.4e-06, |
| "loss": -0.0015, |
| "step": 164 |
| }, |
| { |
| "clip_ratio": 0.0005446214054245502, |
| "completion_length": 112.78125, |
| "epoch": 0.022528672856362644, |
| "grad_norm": 0.07420904189348221, |
| "kl": 0.028630498243728653, |
| "learning_rate": 4.394736842105263e-06, |
| "loss": 0.0003, |
| "num_tokens": 470856.0, |
| "reward": 0.859375, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.859375, |
| "step": 165 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.022665210267613325, |
| "grad_norm": 0.07424714416265488, |
| "kl": 0.02978915546555072, |
| "learning_rate": 4.3894736842105266e-06, |
| "loss": 0.0001, |
| "step": 166 |
| }, |
| { |
| "clip_ratio": 0.0005446214054245502, |
| "epoch": 0.022801747678864007, |
| "grad_norm": 0.0685018002986908, |
| "kl": 0.03043460487970151, |
| "learning_rate": 4.38421052631579e-06, |
| "loss": -0.0003, |
| "step": 167 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.022938285090114693, |
| "grad_norm": 0.06347281485795975, |
| "kl": 0.03080894984304905, |
| "learning_rate": 4.378947368421053e-06, |
| "loss": -0.0009, |
| "step": 168 |
| }, |
| { |
| "clip_ratio": 0.0003324467979837209, |
| "completion_length": 115.125, |
| "epoch": 0.023074822501365375, |
| "grad_norm": 0.09582728147506714, |
| "kl": 0.029706816072575748, |
| "learning_rate": 4.373684210526316e-06, |
| "loss": 0.0005, |
| "num_tokens": 481480.0, |
| "reward": 0.4937500059604645, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.4937500059604645, |
| "step": 169 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.023211359912616056, |
| "grad_norm": 0.08268621563911438, |
| "kl": 0.029470782610587776, |
| "learning_rate": 4.368421052631579e-06, |
| "loss": -0.0, |
| "step": 170 |
| }, |
| { |
| "clip_ratio": 0.00023854961909819394, |
| "epoch": 0.02334789732386674, |
| "grad_norm": 0.07390839606523514, |
| "kl": 0.029934398946352303, |
| "learning_rate": 4.363157894736842e-06, |
| "loss": -0.0005, |
| "step": 171 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.023484434735117424, |
| "grad_norm": 0.07234860211610794, |
| "kl": 0.02987176994793117, |
| "learning_rate": 4.3578947368421055e-06, |
| "loss": -0.0009, |
| "step": 172 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.84375, |
| "epoch": 0.023620972146368106, |
| "grad_norm": 0.07804840058088303, |
| "kl": 0.030131216015433893, |
| "learning_rate": 4.352631578947369e-06, |
| "loss": 0.0002, |
| "num_tokens": 491971.0, |
| "reward": 0.887499988079071, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.887499988079071, |
| "step": 173 |
| }, |
| { |
| "clip_ratio": 0.0005341880605556071, |
| "epoch": 0.023757509557618788, |
| "grad_norm": 0.07273226976394653, |
| "kl": 0.030887642147717997, |
| "learning_rate": 4.347368421052632e-06, |
| "loss": 0.0, |
| "step": 174 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.02389404696886947, |
| "grad_norm": 0.06586578488349915, |
| "kl": 0.031295952590880916, |
| "learning_rate": 4.342105263157895e-06, |
| "loss": -0.0005, |
| "step": 175 |
| }, |
| { |
| "clip_ratio": 0.0008401820377912372, |
| "epoch": 0.02403058438012015, |
| "grad_norm": 0.059170957654714584, |
| "kl": 0.03198954090476036, |
| "learning_rate": 4.336842105263158e-06, |
| "loss": -0.001, |
| "step": 176 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.65625, |
| "epoch": 0.024167121791370837, |
| "grad_norm": 0.08889607340097427, |
| "kl": 0.04427822964498773, |
| "learning_rate": 4.331578947368421e-06, |
| "loss": 0.0006, |
| "num_tokens": 502488.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 177 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.02430365920262152, |
| "grad_norm": 0.07279235869646072, |
| "kl": 0.0449052246985957, |
| "learning_rate": 4.3263157894736845e-06, |
| "loss": 0.0001, |
| "step": 178 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0244401966138722, |
| "grad_norm": 0.072224460542202, |
| "kl": 0.04581551585579291, |
| "learning_rate": 4.321052631578948e-06, |
| "loss": -0.0002, |
| "step": 179 |
| }, |
| { |
| "clip_ratio": 0.00018825300503522158, |
| "epoch": 0.024576734025122882, |
| "grad_norm": 0.06858509033918381, |
| "kl": 0.04448665684321895, |
| "learning_rate": 4.315789473684211e-06, |
| "loss": -0.0008, |
| "step": 180 |
| }, |
| { |
| "clip_ratio": 0.00035919540096074343, |
| "completion_length": 99.4375, |
| "epoch": 0.024713271436373568, |
| "grad_norm": 0.06964289397001266, |
| "kl": 0.03402404289226979, |
| "learning_rate": 4.310526315789474e-06, |
| "loss": 0.0004, |
| "num_tokens": 513246.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 181 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.02484980884762425, |
| "grad_norm": 0.06922803074121475, |
| "kl": 0.03361908288206905, |
| "learning_rate": 4.305263157894737e-06, |
| "loss": 0.0003, |
| "step": 182 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.02498634625887493, |
| "grad_norm": 0.0668816789984703, |
| "kl": 0.033550983702298254, |
| "learning_rate": 4.3e-06, |
| "loss": -0.0, |
| "step": 183 |
| }, |
| { |
| "clip_ratio": 0.0003511235991027206, |
| "epoch": 0.025122883670125613, |
| "grad_norm": 0.05455351620912552, |
| "kl": 0.03327295114286244, |
| "learning_rate": 4.2947368421052635e-06, |
| "loss": -0.0005, |
| "step": 184 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.65625, |
| "epoch": 0.0252594210813763, |
| "grad_norm": 0.11938021332025528, |
| "kl": 0.044678817328531295, |
| "learning_rate": 4.289473684210527e-06, |
| "loss": 0.0005, |
| "num_tokens": 524007.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.2511056959629059, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 185 |
| }, |
| { |
| "clip_ratio": 0.00034722223062999547, |
| "epoch": 0.02539595849262698, |
| "grad_norm": 0.11370915919542313, |
| "kl": 0.04234666272532195, |
| "learning_rate": 4.28421052631579e-06, |
| "loss": -0.0003, |
| "step": 186 |
| }, |
| { |
| "clip_ratio": 0.000662878795992583, |
| "epoch": 0.025532495903877662, |
| "grad_norm": 0.10087863355875015, |
| "kl": 0.04062945069745183, |
| "learning_rate": 4.278947368421053e-06, |
| "loss": -0.001, |
| "step": 187 |
| }, |
| { |
| "clip_ratio": 0.0005541758728213608, |
| "epoch": 0.025669033315128344, |
| "grad_norm": 0.09143723547458649, |
| "kl": 0.0406409723800607, |
| "learning_rate": 4.273684210526316e-06, |
| "loss": -0.0017, |
| "step": 188 |
| }, |
| { |
| "clip_ratio": 0.0005189153016544878, |
| "completion_length": 103.1875, |
| "epoch": 0.025805570726379026, |
| "grad_norm": 0.11307642608880997, |
| "kl": 0.03528324858052656, |
| "learning_rate": 4.268421052631579e-06, |
| "loss": 0.0005, |
| "num_tokens": 533865.0, |
| "reward": 0.7750000357627869, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.7750000357627869, |
| "step": 189 |
| }, |
| { |
| "clip_ratio": 0.0003124999930150807, |
| "epoch": 0.02594210813762971, |
| "grad_norm": 0.1095878928899765, |
| "kl": 0.03600154822925106, |
| "learning_rate": 4.2631578947368425e-06, |
| "loss": -0.0002, |
| "step": 190 |
| }, |
| { |
| "clip_ratio": 0.0006060116575099528, |
| "epoch": 0.026078645548880394, |
| "grad_norm": 0.0969381183385849, |
| "kl": 0.035806038504233584, |
| "learning_rate": 4.257894736842106e-06, |
| "loss": -0.0011, |
| "step": 191 |
| }, |
| { |
| "clip_ratio": 0.0005024050769861788, |
| "epoch": 0.026215182960131075, |
| "grad_norm": 0.08637399971485138, |
| "kl": 0.03535072586964816, |
| "learning_rate": 4.252631578947369e-06, |
| "loss": -0.0018, |
| "step": 192 |
| }, |
| { |
| "completion_length": 106.0, |
| "epoch": 0.026351720371381757, |
| "grad_norm": 0.0, |
| "learning_rate": 4.247368421052632e-06, |
| "loss": 0.0, |
| "num_tokens": 543893.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.0, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 193 |
| }, |
| { |
| "epoch": 0.026488257782632443, |
| "grad_norm": 0.0, |
| "learning_rate": 4.242105263157895e-06, |
| "loss": 0.0, |
| "step": 194 |
| }, |
| { |
| "epoch": 0.026624795193883125, |
| "grad_norm": 0.0, |
| "learning_rate": 4.236842105263158e-06, |
| "loss": 0.0, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.026761332605133806, |
| "grad_norm": 0.0, |
| "learning_rate": 4.2315789473684215e-06, |
| "loss": 0.0, |
| "step": 196 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.1875, |
| "epoch": 0.02689787001638449, |
| "grad_norm": 0.10188470780849457, |
| "kl": 0.03902641427703202, |
| "learning_rate": 4.226315789473685e-06, |
| "loss": 0.0002, |
| "num_tokens": 554083.0, |
| "reward": 0.859375, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.859375, |
| "step": 197 |
| }, |
| { |
| "clip_ratio": 0.00021404109429568052, |
| "epoch": 0.027034407427635174, |
| "grad_norm": 0.09636596590280533, |
| "kl": 0.03859919391106814, |
| "learning_rate": 4.221052631578948e-06, |
| "loss": 0.0, |
| "step": 198 |
| }, |
| { |
| "clip_ratio": 0.00045628915540874004, |
| "epoch": 0.027170944838885856, |
| "grad_norm": 0.07624981552362442, |
| "kl": 0.03810006860294379, |
| "learning_rate": 4.215789473684211e-06, |
| "loss": -0.0007, |
| "step": 199 |
| }, |
| { |
| "clip_ratio": 0.0005362060328479856, |
| "epoch": 0.027307482250136537, |
| "grad_norm": 0.07269936800003052, |
| "kl": 0.038183362572453916, |
| "learning_rate": 4.210526315789474e-06, |
| "loss": -0.0012, |
| "step": 200 |
| }, |
| { |
| "clip_ratio": 0.0004032258002553135, |
| "completion_length": 105.375, |
| "epoch": 0.02744401966138722, |
| "grad_norm": 0.06504663079977036, |
| "kl": 0.03330190840642899, |
| "learning_rate": 4.205263157894737e-06, |
| "loss": 0.0003, |
| "num_tokens": 564759.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 201 |
| }, |
| { |
| "clip_ratio": 0.0004032258002553135, |
| "epoch": 0.0275805570726379, |
| "grad_norm": 0.0650041475892067, |
| "kl": 0.03405800275504589, |
| "learning_rate": 4.2000000000000004e-06, |
| "loss": 0.0001, |
| "step": 202 |
| }, |
| { |
| "clip_ratio": 0.0006048387149348855, |
| "epoch": 0.027717094483888587, |
| "grad_norm": 0.06338495761156082, |
| "kl": 0.03414264236926101, |
| "learning_rate": 4.194736842105264e-06, |
| "loss": -0.0002, |
| "step": 203 |
| }, |
| { |
| "clip_ratio": 0.0004032258002553135, |
| "epoch": 0.02785363189513927, |
| "grad_norm": 0.06049591302871704, |
| "kl": 0.03434641310013831, |
| "learning_rate": 4.189473684210527e-06, |
| "loss": -0.0007, |
| "step": 204 |
| }, |
| { |
| "clip_ratio": 0.0007811779651092365, |
| "completion_length": 117.34375, |
| "epoch": 0.02799016930638995, |
| "grad_norm": 0.09299098700284958, |
| "kl": 0.03305593744153157, |
| "learning_rate": 4.18421052631579e-06, |
| "loss": 0.0004, |
| "num_tokens": 575498.0, |
| "reward": 0.4937500059604645, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.4937500059604645, |
| "step": 205 |
| }, |
| { |
| "clip_ratio": 0.0003324467979837209, |
| "epoch": 0.028126706717640632, |
| "grad_norm": 0.09308190643787384, |
| "kl": 0.03269680918310769, |
| "learning_rate": 4.178947368421053e-06, |
| "loss": 0.0003, |
| "step": 206 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.028263244128891318, |
| "grad_norm": 0.08965054899454117, |
| "kl": 0.03241226324462332, |
| "learning_rate": 4.173684210526316e-06, |
| "loss": -0.0006, |
| "step": 207 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.028399781540142, |
| "grad_norm": 0.07739660888910294, |
| "kl": 0.03276632426423021, |
| "learning_rate": 4.1684210526315794e-06, |
| "loss": -0.0013, |
| "step": 208 |
| }, |
| { |
| "clip_ratio": 0.00019409938249737024, |
| "completion_length": 123.375, |
| "epoch": 0.02853631895139268, |
| "grad_norm": 0.06903862208127975, |
| "kl": 0.02725412551080808, |
| "learning_rate": 4.163157894736843e-06, |
| "loss": 0.0004, |
| "num_tokens": 587282.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 209 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.028672856362643363, |
| "grad_norm": 0.06824494898319244, |
| "kl": 0.027433462004410103, |
| "learning_rate": 4.157894736842106e-06, |
| "loss": 0.0002, |
| "step": 210 |
| }, |
| { |
| "clip_ratio": 0.0004374573181848973, |
| "epoch": 0.02880939377389405, |
| "grad_norm": 0.06868710368871689, |
| "kl": 0.02734687612974085, |
| "learning_rate": 4.152631578947369e-06, |
| "loss": -0.0001, |
| "step": 211 |
| }, |
| { |
| "clip_ratio": 0.00023320894979406148, |
| "epoch": 0.02894593118514473, |
| "grad_norm": 0.0652066022157669, |
| "kl": 0.027320355700794607, |
| "learning_rate": 4.147368421052632e-06, |
| "loss": -0.0008, |
| "step": 212 |
| }, |
| { |
| "clip_ratio": 0.00042826301068998873, |
| "completion_length": 126.1875, |
| "epoch": 0.029082468596395412, |
| "grad_norm": 0.04128393158316612, |
| "kl": 0.027315025363350287, |
| "learning_rate": 4.142105263157895e-06, |
| "loss": 0.0003, |
| "num_tokens": 598840.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 213 |
| }, |
| { |
| "clip_ratio": 0.0008565260213799775, |
| "epoch": 0.029219006007646094, |
| "grad_norm": 0.040705274790525436, |
| "kl": 0.02696406643372029, |
| "learning_rate": 4.136842105263158e-06, |
| "loss": 0.0003, |
| "step": 214 |
| }, |
| { |
| "clip_ratio": 0.0004370629321783781, |
| "epoch": 0.029355543418896776, |
| "grad_norm": 0.0406440831720829, |
| "kl": 0.02672984532546252, |
| "learning_rate": 4.1315789473684216e-06, |
| "loss": 0.0001, |
| "step": 215 |
| }, |
| { |
| "clip_ratio": 0.0006467944767791778, |
| "epoch": 0.02949208083014746, |
| "grad_norm": 0.039546553045511246, |
| "kl": 0.026357230555731803, |
| "learning_rate": 4.126315789473685e-06, |
| "loss": -0.0001, |
| "step": 216 |
| }, |
| { |
| "clip_ratio": 0.00021551724057644606, |
| "completion_length": 125.65625, |
| "epoch": 0.029628618241398143, |
| "grad_norm": 0.09606111794710159, |
| "kl": 0.027327494870405644, |
| "learning_rate": 4.121052631578948e-06, |
| "loss": 0.0001, |
| "num_tokens": 610221.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 217 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.029765155652648825, |
| "grad_norm": 0.09475074708461761, |
| "kl": 0.026967918034642935, |
| "learning_rate": 4.115789473684211e-06, |
| "loss": -0.0003, |
| "step": 218 |
| }, |
| { |
| "clip_ratio": 0.00021551724057644606, |
| "epoch": 0.029901693063899507, |
| "grad_norm": 0.09437484294176102, |
| "kl": 0.02764826436759904, |
| "learning_rate": 4.110526315789474e-06, |
| "loss": -0.0009, |
| "step": 219 |
| }, |
| { |
| "clip_ratio": 0.00021551724057644606, |
| "epoch": 0.030038230475150193, |
| "grad_norm": 0.08844147622585297, |
| "kl": 0.02761025112704374, |
| "learning_rate": 4.105263157894737e-06, |
| "loss": -0.0017, |
| "step": 220 |
| }, |
| { |
| "clip_ratio": 0.0004921259824186563, |
| "completion_length": 117.5625, |
| "epoch": 0.030174767886400874, |
| "grad_norm": 0.08956988155841827, |
| "kl": 0.027353109326213598, |
| "learning_rate": 4.1e-06, |
| "loss": 0.0005, |
| "num_tokens": 621283.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.25110572576522827, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 221 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.030311305297651556, |
| "grad_norm": 0.09050950407981873, |
| "kl": 0.02785796401440166, |
| "learning_rate": 4.094736842105264e-06, |
| "loss": -0.0003, |
| "step": 222 |
| }, |
| { |
| "clip_ratio": 0.00039178448787424713, |
| "epoch": 0.030447842708902238, |
| "grad_norm": 0.08440433442592621, |
| "kl": 0.0283550031890627, |
| "learning_rate": 4.089473684210527e-06, |
| "loss": -0.0006, |
| "step": 223 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.030584380120152924, |
| "grad_norm": 0.078850157558918, |
| "kl": 0.029308805766049773, |
| "learning_rate": 4.08421052631579e-06, |
| "loss": -0.0015, |
| "step": 224 |
| }, |
| { |
| "clip_ratio": 0.000794308609329164, |
| "completion_length": 112.1875, |
| "epoch": 0.030720917531403606, |
| "grad_norm": 0.09105496853590012, |
| "kl": 0.02974900306435302, |
| "learning_rate": 4.078947368421053e-06, |
| "loss": 0.0002, |
| "num_tokens": 631645.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 225 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.030857454942654287, |
| "grad_norm": 0.10003118962049484, |
| "kl": 0.029050659737549722, |
| "learning_rate": 4.073684210526316e-06, |
| "loss": -0.0, |
| "step": 226 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03099399235390497, |
| "grad_norm": 0.08923932909965515, |
| "kl": 0.02833891040063463, |
| "learning_rate": 4.0684210526315795e-06, |
| "loss": -0.0006, |
| "step": 227 |
| }, |
| { |
| "clip_ratio": 0.0015976974100340158, |
| "epoch": 0.03113052976515565, |
| "grad_norm": 0.07948505878448486, |
| "kl": 0.02925992920063436, |
| "learning_rate": 4.063157894736842e-06, |
| "loss": -0.0015, |
| "step": 228 |
| }, |
| { |
| "clip_ratio": 0.0005824159306939691, |
| "completion_length": 112.875, |
| "epoch": 0.03126706717640634, |
| "grad_norm": 0.08888618648052216, |
| "kl": 0.026295807358110324, |
| "learning_rate": 4.057894736842106e-06, |
| "loss": 0.0004, |
| "num_tokens": 642929.0, |
| "reward": 0.578125, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.578125, |
| "step": 229 |
| }, |
| { |
| "clip_ratio": 0.001052564155543223, |
| "epoch": 0.031403604587657015, |
| "grad_norm": 0.08594765514135361, |
| "kl": 0.026338903553551063, |
| "learning_rate": 4.052631578947368e-06, |
| "loss": -0.0002, |
| "step": 230 |
| }, |
| { |
| "clip_ratio": 0.0005595455877482891, |
| "epoch": 0.0315401419989077, |
| "grad_norm": 0.08271999657154083, |
| "kl": 0.026568240544293076, |
| "learning_rate": 4.047368421052632e-06, |
| "loss": -0.0006, |
| "step": 231 |
| }, |
| { |
| "clip_ratio": 0.0005595455877482891, |
| "epoch": 0.031676679410158386, |
| "grad_norm": 0.07526501268148422, |
| "kl": 0.026544391934294254, |
| "learning_rate": 4.042105263157895e-06, |
| "loss": -0.0014, |
| "step": 232 |
| }, |
| { |
| "clip_ratio": 0.0007640966796316206, |
| "completion_length": 113.625, |
| "epoch": 0.031813216821409064, |
| "grad_norm": 0.0923089012503624, |
| "kl": 0.024078507034573704, |
| "learning_rate": 4.0368421052631585e-06, |
| "loss": 0.0003, |
| "num_tokens": 653789.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 233 |
| }, |
| { |
| "clip_ratio": 0.0004877125611528754, |
| "epoch": 0.03194975423265975, |
| "grad_norm": 0.09331841766834259, |
| "kl": 0.024048013438005, |
| "learning_rate": 4.031578947368422e-06, |
| "loss": -0.0001, |
| "step": 234 |
| }, |
| { |
| "clip_ratio": 0.00019290123600512743, |
| "epoch": 0.032086291643910435, |
| "grad_norm": 0.08464116603136063, |
| "kl": 0.023768270009895787, |
| "learning_rate": 4.026315789473684e-06, |
| "loss": -0.0008, |
| "step": 235 |
| }, |
| { |
| "clip_ratio": 0.0004210034239804372, |
| "epoch": 0.03222282905516111, |
| "grad_norm": 0.07866553962230682, |
| "kl": 0.023468335362849757, |
| "learning_rate": 4.021052631578948e-06, |
| "loss": -0.0016, |
| "step": 236 |
| }, |
| { |
| "clip_ratio": 0.00018274853937327862, |
| "completion_length": 113.0, |
| "epoch": 0.0323593664664118, |
| "grad_norm": 0.06890566647052765, |
| "kl": 0.02587902924278751, |
| "learning_rate": 4.01578947368421e-06, |
| "loss": 0.0004, |
| "num_tokens": 664541.0, |
| "reward": 0.4375, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.4375, |
| "step": 237 |
| }, |
| { |
| "clip_ratio": 0.0003633720916695893, |
| "epoch": 0.03249590387766248, |
| "grad_norm": 0.06925798207521439, |
| "kl": 0.025461589102633297, |
| "learning_rate": 4.010526315789474e-06, |
| "loss": 0.0002, |
| "step": 238 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03263244128891316, |
| "grad_norm": 0.06528756767511368, |
| "kl": 0.02574267768068239, |
| "learning_rate": 4.005263157894737e-06, |
| "loss": 0.0, |
| "step": 239 |
| }, |
| { |
| "clip_ratio": 0.00018274853937327862, |
| "epoch": 0.03276897870016385, |
| "grad_norm": 0.06375279277563095, |
| "kl": 0.025849525845842436, |
| "learning_rate": 4.000000000000001e-06, |
| "loss": -0.0004, |
| "step": 240 |
| }, |
| { |
| "clip_ratio": 0.0008375860925298184, |
| "completion_length": 106.09375, |
| "epoch": 0.032905516111414526, |
| "grad_norm": 0.0751417875289917, |
| "kl": 0.027247032237937674, |
| "learning_rate": 3.994736842105264e-06, |
| "loss": 0.0004, |
| "num_tokens": 674724.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 241 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03304205352266521, |
| "grad_norm": 0.07292292267084122, |
| "kl": 0.02736963256029412, |
| "learning_rate": 3.989473684210526e-06, |
| "loss": 0.0, |
| "step": 242 |
| }, |
| { |
| "clip_ratio": 0.0003033980610780418, |
| "epoch": 0.03317859093391589, |
| "grad_norm": 0.06825637817382812, |
| "kl": 0.027891719597391784, |
| "learning_rate": 3.98421052631579e-06, |
| "loss": -0.0003, |
| "step": 243 |
| }, |
| { |
| "clip_ratio": 0.0003033980610780418, |
| "epoch": 0.033315128345166575, |
| "grad_norm": 0.06375395506620407, |
| "kl": 0.0283044419484213, |
| "learning_rate": 3.9789473684210525e-06, |
| "loss": -0.0007, |
| "step": 244 |
| }, |
| { |
| "clip_ratio": 0.0005482456181198359, |
| "completion_length": 124.5625, |
| "epoch": 0.03345166575641726, |
| "grad_norm": 0.09664054214954376, |
| "kl": 0.028416587243555114, |
| "learning_rate": 3.9736842105263165e-06, |
| "loss": 0.0002, |
| "num_tokens": 685466.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 245 |
| }, |
| { |
| "clip_ratio": 0.0003923962212866172, |
| "epoch": 0.03358820316766794, |
| "grad_norm": 0.09948284178972244, |
| "kl": 0.02821255396702327, |
| "learning_rate": 3.968421052631579e-06, |
| "loss": -0.0, |
| "step": 246 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.033724740578918624, |
| "grad_norm": 0.07397827506065369, |
| "kl": 0.02847255891538225, |
| "learning_rate": 3.963157894736843e-06, |
| "loss": -0.0006, |
| "step": 247 |
| }, |
| { |
| "clip_ratio": 0.0004448878316907212, |
| "epoch": 0.03386127799016931, |
| "grad_norm": 0.07055655121803284, |
| "kl": 0.028869623522041366, |
| "learning_rate": 3.957894736842106e-06, |
| "loss": -0.0011, |
| "step": 248 |
| }, |
| { |
| "clip_ratio": 0.0002765486715361476, |
| "completion_length": 94.875, |
| "epoch": 0.03399781540141999, |
| "grad_norm": 0.052606552839279175, |
| "kl": 0.020315775473136455, |
| "learning_rate": 3.952631578947368e-06, |
| "loss": 0.0004, |
| "num_tokens": 696522.0, |
| "reward": 0.831250011920929, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.831250011920929, |
| "step": 249 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.034134352812670674, |
| "grad_norm": 0.05249060317873955, |
| "kl": 0.020589739870047197, |
| "learning_rate": 3.947368421052632e-06, |
| "loss": 0.0001, |
| "step": 250 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03427089022392135, |
| "grad_norm": 0.04982711747288704, |
| "kl": 0.02040288085117936, |
| "learning_rate": 3.942105263157895e-06, |
| "loss": 0.0, |
| "step": 251 |
| }, |
| { |
| "clip_ratio": 0.0002765486715361476, |
| "epoch": 0.03440742763517204, |
| "grad_norm": 0.04824934899806976, |
| "kl": 0.020667625038186088, |
| "learning_rate": 3.936842105263159e-06, |
| "loss": -0.0002, |
| "step": 252 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.34375, |
| "epoch": 0.03454396504642272, |
| "grad_norm": 0.057229530066251755, |
| "kl": 0.03170221848995425, |
| "learning_rate": 3.931578947368421e-06, |
| "loss": 0.0003, |
| "num_tokens": 707233.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 253 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0346805024576734, |
| "grad_norm": 0.057189639657735825, |
| "kl": 0.03171565095544793, |
| "learning_rate": 3.926315789473685e-06, |
| "loss": 0.0, |
| "step": 254 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.034817039868924086, |
| "grad_norm": 0.05530928447842598, |
| "kl": 0.030918381758965552, |
| "learning_rate": 3.921052631578947e-06, |
| "loss": -0.0002, |
| "step": 255 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.034953577280174765, |
| "grad_norm": 0.052288055419921875, |
| "kl": 0.03107694143545814, |
| "learning_rate": 3.9157894736842104e-06, |
| "loss": -0.0004, |
| "step": 256 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.34375, |
| "epoch": 0.03509011469142545, |
| "grad_norm": 0.0891910195350647, |
| "kl": 0.023110102163627744, |
| "learning_rate": 3.9105263157894744e-06, |
| "loss": 0.0001, |
| "num_tokens": 718136.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 257 |
| }, |
| { |
| "clip_ratio": 0.00022163119865581393, |
| "epoch": 0.035226652102676136, |
| "grad_norm": 0.07997400313615799, |
| "kl": 0.02337674022419378, |
| "learning_rate": 3.905263157894737e-06, |
| "loss": -0.0002, |
| "step": 258 |
| }, |
| { |
| "clip_ratio": 0.00020695364219136536, |
| "epoch": 0.035363189513926814, |
| "grad_norm": 0.0842948779463768, |
| "kl": 0.022346210171235725, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": -0.0006, |
| "step": 259 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0354997269251775, |
| "grad_norm": 0.07676080614328384, |
| "kl": 0.02255020069424063, |
| "learning_rate": 3.894736842105263e-06, |
| "loss": -0.0012, |
| "step": 260 |
| }, |
| { |
| "clip_ratio": 0.0008840422669891268, |
| "completion_length": 102.75, |
| "epoch": 0.035636264336428185, |
| "grad_norm": 0.12544368207454681, |
| "kl": 0.02338628371944651, |
| "learning_rate": 3.889473684210527e-06, |
| "loss": 0.0005, |
| "num_tokens": 728780.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 261 |
| }, |
| { |
| "clip_ratio": 0.000634926778730005, |
| "epoch": 0.03577280174767886, |
| "grad_norm": 0.11735007166862488, |
| "kl": 0.023411447560647503, |
| "learning_rate": 3.884210526315789e-06, |
| "loss": -0.0002, |
| "step": 262 |
| }, |
| { |
| "clip_ratio": 0.00048496242379769683, |
| "epoch": 0.03590933915892955, |
| "grad_norm": 0.11159281432628632, |
| "kl": 0.023148178006522357, |
| "learning_rate": 3.878947368421053e-06, |
| "loss": -0.001, |
| "step": 263 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03604587657018023, |
| "grad_norm": 0.09994325041770935, |
| "kl": 0.023549747798824683, |
| "learning_rate": 3.873684210526316e-06, |
| "loss": -0.0021, |
| "step": 264 |
| }, |
| { |
| "clip_ratio": 0.0004883110523223877, |
| "completion_length": 115.15625, |
| "epoch": 0.03618241398143091, |
| "grad_norm": 0.07745163142681122, |
| "kl": 0.026939595787553117, |
| "learning_rate": 3.868421052631579e-06, |
| "loss": 0.0003, |
| "num_tokens": 739625.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 265 |
| }, |
| { |
| "clip_ratio": 0.00029761905898340046, |
| "epoch": 0.0363189513926816, |
| "grad_norm": 0.07724476605653763, |
| "kl": 0.027008358330931515, |
| "learning_rate": 3.863157894736843e-06, |
| "loss": 0.0, |
| "step": 266 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.036455488803932276, |
| "grad_norm": 0.07575860619544983, |
| "kl": 0.027253914857283235, |
| "learning_rate": 3.857894736842105e-06, |
| "loss": -0.0004, |
| "step": 267 |
| }, |
| { |
| "clip_ratio": 0.0005524969019461423, |
| "epoch": 0.03659202621518296, |
| "grad_norm": 0.07534480094909668, |
| "kl": 0.02772324366378598, |
| "learning_rate": 3.852631578947369e-06, |
| "loss": -0.0008, |
| "step": 268 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.5625, |
| "epoch": 0.03672856362643364, |
| "grad_norm": 0.03475145250558853, |
| "kl": 0.021824754105182365, |
| "learning_rate": 3.8473684210526316e-06, |
| "loss": 0.0002, |
| "num_tokens": 750175.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 269 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.036865101037684325, |
| "grad_norm": 0.03451026603579521, |
| "kl": 0.02219120250083506, |
| "learning_rate": 3.842105263157895e-06, |
| "loss": 0.0002, |
| "step": 270 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03700163844893501, |
| "grad_norm": 0.03484285995364189, |
| "kl": 0.022240082558710128, |
| "learning_rate": 3.836842105263158e-06, |
| "loss": 0.0002, |
| "step": 271 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03713817586018569, |
| "grad_norm": 0.035136640071868896, |
| "kl": 0.022295867995126173, |
| "learning_rate": 3.831578947368421e-06, |
| "loss": -0.0001, |
| "step": 272 |
| }, |
| { |
| "clip_ratio": 0.00013412017142400146, |
| "completion_length": 121.78125, |
| "epoch": 0.037274713271436374, |
| "grad_norm": 0.09162586182355881, |
| "kl": 0.02208938094554469, |
| "learning_rate": 3.826315789473685e-06, |
| "loss": 0.0, |
| "num_tokens": 761652.0, |
| "reward": 0.887499988079071, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.887499988079071, |
| "step": 273 |
| }, |
| { |
| "clip_ratio": 0.00025201612152159214, |
| "epoch": 0.03741125068268706, |
| "grad_norm": 0.08879063278436661, |
| "kl": 0.02217887411825359, |
| "learning_rate": 3.821052631578947e-06, |
| "loss": -0.0002, |
| "step": 274 |
| }, |
| { |
| "clip_ratio": 0.0007802476466167718, |
| "epoch": 0.03754778809393774, |
| "grad_norm": 0.08000439405441284, |
| "kl": 0.022196775797056034, |
| "learning_rate": 3.815789473684211e-06, |
| "loss": -0.0006, |
| "step": 275 |
| }, |
| { |
| "clip_ratio": 0.0007268308690981939, |
| "epoch": 0.037684325505188423, |
| "grad_norm": 0.06881428509950638, |
| "kl": 0.02263434650376439, |
| "learning_rate": 3.810526315789474e-06, |
| "loss": -0.0013, |
| "step": 276 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 120.28125, |
| "epoch": 0.0378208629164391, |
| "grad_norm": 0.05088047683238983, |
| "kl": 0.02204109003650956, |
| "learning_rate": 3.805263157894737e-06, |
| "loss": 0.0003, |
| "num_tokens": 773309.0, |
| "reward": 0.859375, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.859375, |
| "step": 277 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03795740032768979, |
| "grad_norm": 0.049479130655527115, |
| "kl": 0.022138332104077563, |
| "learning_rate": 3.8000000000000005e-06, |
| "loss": 0.0, |
| "step": 278 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03809393773894047, |
| "grad_norm": 0.0442269891500473, |
| "kl": 0.022356969930115156, |
| "learning_rate": 3.794736842105263e-06, |
| "loss": -0.0001, |
| "step": 279 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03823047515019115, |
| "grad_norm": 0.04249510541558266, |
| "kl": 0.022337044501909986, |
| "learning_rate": 3.789473684210527e-06, |
| "loss": -0.0003, |
| "step": 280 |
| }, |
| { |
| "clip_ratio": 0.0005657151050399989, |
| "completion_length": 113.625, |
| "epoch": 0.038367012561441836, |
| "grad_norm": 0.07635533064603806, |
| "kl": 0.023303329100599512, |
| "learning_rate": 3.7842105263157895e-06, |
| "loss": 0.0003, |
| "num_tokens": 784297.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 281 |
| }, |
| { |
| "clip_ratio": 0.00027901786961592734, |
| "epoch": 0.038503549972692515, |
| "grad_norm": 0.0776119977235794, |
| "kl": 0.02313061192398891, |
| "learning_rate": 3.778947368421053e-06, |
| "loss": 0.0002, |
| "step": 282 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0386400873839432, |
| "grad_norm": 0.0766444057226181, |
| "kl": 0.02328120105084963, |
| "learning_rate": 3.773684210526316e-06, |
| "loss": -0.0003, |
| "step": 283 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.038776624795193886, |
| "grad_norm": 0.0710226446390152, |
| "kl": 0.023574423365062103, |
| "learning_rate": 3.768421052631579e-06, |
| "loss": -0.001, |
| "step": 284 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.0625, |
| "epoch": 0.038913162206444564, |
| "grad_norm": 0.061644088476896286, |
| "kl": 0.02391217538388446, |
| "learning_rate": 3.7631578947368426e-06, |
| "loss": 0.0003, |
| "num_tokens": 795303.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 285 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03904969961769525, |
| "grad_norm": 0.0605340413749218, |
| "kl": 0.0234130252210889, |
| "learning_rate": 3.7578947368421053e-06, |
| "loss": 0.0001, |
| "step": 286 |
| }, |
| { |
| "clip_ratio": 0.0002460629912093282, |
| "epoch": 0.03918623702894593, |
| "grad_norm": 0.060991305857896805, |
| "kl": 0.02373662660829723, |
| "learning_rate": 3.752631578947369e-06, |
| "loss": 0.0, |
| "step": 287 |
| }, |
| { |
| "clip_ratio": 0.0002460629912093282, |
| "epoch": 0.03932277444019661, |
| "grad_norm": 0.054490797221660614, |
| "kl": 0.02412348723737523, |
| "learning_rate": 3.7473684210526317e-06, |
| "loss": -0.0006, |
| "step": 288 |
| }, |
| { |
| "clip_ratio": 0.00044014083687216043, |
| "completion_length": 122.59375, |
| "epoch": 0.0394593118514473, |
| "grad_norm": 0.07991161197423935, |
| "kl": 0.022707521973643452, |
| "learning_rate": 3.7421052631578953e-06, |
| "loss": 0.0004, |
| "num_tokens": 806314.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 289 |
| }, |
| { |
| "clip_ratio": 0.00022007041843608022, |
| "epoch": 0.03959584926269798, |
| "grad_norm": 0.07932053506374359, |
| "kl": 0.022475473844679072, |
| "learning_rate": 3.736842105263158e-06, |
| "loss": -0.0001, |
| "step": 290 |
| }, |
| { |
| "clip_ratio": 0.00022007041843608022, |
| "epoch": 0.03973238667394866, |
| "grad_norm": 0.07653678953647614, |
| "kl": 0.022855573624838144, |
| "learning_rate": 3.7315789473684216e-06, |
| "loss": -0.0003, |
| "step": 291 |
| }, |
| { |
| "clip_ratio": 0.0003703108086483553, |
| "epoch": 0.03986892408519935, |
| "grad_norm": 0.06487143039703369, |
| "kl": 0.023218183137942106, |
| "learning_rate": 3.7263157894736848e-06, |
| "loss": -0.0008, |
| "step": 292 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 120.3125, |
| "epoch": 0.040005461496450026, |
| "grad_norm": 0.05640777572989464, |
| "kl": 0.021730259235482663, |
| "learning_rate": 3.7210526315789475e-06, |
| "loss": 0.0003, |
| "num_tokens": 817436.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 293 |
| }, |
| { |
| "clip_ratio": 0.0005408743163570762, |
| "epoch": 0.04014199890770071, |
| "grad_norm": 0.05590509995818138, |
| "kl": 0.021133837261004373, |
| "learning_rate": 3.715789473684211e-06, |
| "loss": 0.0003, |
| "step": 294 |
| }, |
| { |
| "clip_ratio": 0.0002460629912093282, |
| "epoch": 0.04027853631895139, |
| "grad_norm": 0.05611417070031166, |
| "kl": 0.021910520212259144, |
| "learning_rate": 3.710526315789474e-06, |
| "loss": 0.0002, |
| "step": 295 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.040415073730202075, |
| "grad_norm": 0.05293579027056694, |
| "kl": 0.021879399777390063, |
| "learning_rate": 3.7052631578947374e-06, |
| "loss": -0.0002, |
| "step": 296 |
| }, |
| { |
| "clip_ratio": 0.00021853146608918905, |
| "completion_length": 119.3125, |
| "epoch": 0.04055161114145276, |
| "grad_norm": 0.08654630929231644, |
| "kl": 0.022178779676323757, |
| "learning_rate": 3.7e-06, |
| "loss": 0.0, |
| "num_tokens": 828762.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 297 |
| }, |
| { |
| "clip_ratio": 0.0004370629321783781, |
| "epoch": 0.04068814855270344, |
| "grad_norm": 0.08861983567476273, |
| "kl": 0.022127668256871402, |
| "learning_rate": 3.6947368421052637e-06, |
| "loss": -0.0001, |
| "step": 298 |
| }, |
| { |
| "clip_ratio": 0.0004370629321783781, |
| "epoch": 0.040824685963954124, |
| "grad_norm": 0.08493945002555847, |
| "kl": 0.021997908304911107, |
| "learning_rate": 3.6894736842105265e-06, |
| "loss": -0.0006, |
| "step": 299 |
| }, |
| { |
| "clip_ratio": 0.00021853146608918905, |
| "epoch": 0.0409612233752048, |
| "grad_norm": 0.07281088829040527, |
| "kl": 0.02197305968729779, |
| "learning_rate": 3.6842105263157896e-06, |
| "loss": -0.0013, |
| "step": 300 |
| }, |
| { |
| "clip_ratio": 0.0004409907996887341, |
| "completion_length": 113.25, |
| "epoch": 0.04109776078645549, |
| "grad_norm": 0.07107827067375183, |
| "kl": 0.022476364712929353, |
| "learning_rate": 3.6789473684210532e-06, |
| "loss": 0.0002, |
| "num_tokens": 840026.0, |
| "reward": 0.859375, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.859375, |
| "step": 301 |
| }, |
| { |
| "clip_ratio": 0.0004409907996887341, |
| "epoch": 0.04123429819770617, |
| "grad_norm": 0.06990018486976624, |
| "kl": 0.02247147186426446, |
| "learning_rate": 3.673684210526316e-06, |
| "loss": 0.0001, |
| "step": 302 |
| }, |
| { |
| "clip_ratio": 0.0004409907996887341, |
| "epoch": 0.04137083560895685, |
| "grad_norm": 0.07051187008619308, |
| "kl": 0.022447484341682866, |
| "learning_rate": 3.6684210526315796e-06, |
| "loss": -0.0002, |
| "step": 303 |
| }, |
| { |
| "clip_ratio": 0.0002367424312978983, |
| "epoch": 0.04150737302020754, |
| "grad_norm": 0.06450870633125305, |
| "kl": 0.022808179928688332, |
| "learning_rate": 3.6631578947368423e-06, |
| "loss": -0.0008, |
| "step": 304 |
| }, |
| { |
| "completion_length": 99.625, |
| "epoch": 0.04164391043145822, |
| "grad_norm": 0.0, |
| "learning_rate": 3.657894736842106e-06, |
| "loss": 0.0, |
| "num_tokens": 849910.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn": 1.0, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.0417804478427089, |
| "grad_norm": 0.0, |
| "learning_rate": 3.6526315789473686e-06, |
| "loss": 0.0, |
| "step": 306 |
| }, |
| { |
| "epoch": 0.041916985253959586, |
| "grad_norm": 0.0, |
| "learning_rate": 3.6473684210526318e-06, |
| "loss": 0.0, |
| "step": 307 |
| }, |
| { |
| "epoch": 0.042053522665210265, |
| "grad_norm": 0.0, |
| "learning_rate": 3.642105263157895e-06, |
| "loss": 0.0, |
| "step": 308 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 124.5625, |
| "epoch": 0.04219006007646095, |
| "grad_norm": 0.04516865313053131, |
| "kl": 0.023450861306628212, |
| "learning_rate": 3.636842105263158e-06, |
| "loss": 0.0002, |
| "num_tokens": 861076.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 309 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.042326597487711635, |
| "grad_norm": 0.044763803482055664, |
| "kl": 0.023947950976435095, |
| "learning_rate": 3.6315789473684217e-06, |
| "loss": 0.0001, |
| "step": 310 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.042463134898962314, |
| "grad_norm": 0.045122891664505005, |
| "kl": 0.02412795228883624, |
| "learning_rate": 3.6263157894736844e-06, |
| "loss": -0.0001, |
| "step": 311 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.042599672310213, |
| "grad_norm": 0.04445788636803627, |
| "kl": 0.02396708173910156, |
| "learning_rate": 3.621052631578948e-06, |
| "loss": -0.0002, |
| "step": 312 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.5625, |
| "epoch": 0.04273620972146368, |
| "grad_norm": 0.127179816365242, |
| "kl": 0.028220025211339816, |
| "learning_rate": 3.6157894736842108e-06, |
| "loss": 0.0005, |
| "num_tokens": 871366.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 313 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04287274713271436, |
| "grad_norm": 0.08750166743993759, |
| "kl": 0.027303016162477434, |
| "learning_rate": 3.610526315789474e-06, |
| "loss": 0.0, |
| "step": 314 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04300928454396505, |
| "grad_norm": 0.08059285581111908, |
| "kl": 0.027014906285330653, |
| "learning_rate": 3.605263157894737e-06, |
| "loss": -0.0003, |
| "step": 315 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04314582195521573, |
| "grad_norm": 0.07490835338830948, |
| "kl": 0.02650223023374565, |
| "learning_rate": 3.6000000000000003e-06, |
| "loss": -0.001, |
| "step": 316 |
| }, |
| { |
| "clip_ratio": 0.00024224806111305952, |
| "completion_length": 124.59375, |
| "epoch": 0.04328235936646641, |
| "grad_norm": 0.08907640725374222, |
| "kl": 0.022105894546257332, |
| "learning_rate": 3.5947368421052634e-06, |
| "loss": -0.0, |
| "num_tokens": 883057.0, |
| "reward": 0.859375, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.859375, |
| "step": 317 |
| }, |
| { |
| "clip_ratio": 0.000507078570080921, |
| "epoch": 0.0434188967777171, |
| "grad_norm": 0.08827097713947296, |
| "kl": 0.021878067345824093, |
| "learning_rate": 3.5894736842105266e-06, |
| "loss": -0.0001, |
| "step": 318 |
| }, |
| { |
| "clip_ratio": 0.00024224806111305952, |
| "epoch": 0.043555434188967776, |
| "grad_norm": 0.08430935442447662, |
| "kl": 0.02233501960290596, |
| "learning_rate": 3.58421052631579e-06, |
| "loss": -0.0008, |
| "step": 319 |
| }, |
| { |
| "clip_ratio": 0.0010449249821249396, |
| "epoch": 0.04369197160021846, |
| "grad_norm": 0.07642368972301483, |
| "kl": 0.022479526698589325, |
| "learning_rate": 3.578947368421053e-06, |
| "loss": -0.0013, |
| "step": 320 |
| }, |
| { |
| "clip_ratio": 0.0004036639875266701, |
| "completion_length": 128.375, |
| "epoch": 0.04382850901146914, |
| "grad_norm": 0.08837485313415527, |
| "kl": 0.029423846543068066, |
| "learning_rate": 3.5736842105263157e-06, |
| "loss": 0.0001, |
| "num_tokens": 894553.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 321 |
| }, |
| { |
| "clip_ratio": 0.0004036639875266701, |
| "epoch": 0.043965046422719825, |
| "grad_norm": 0.08810171484947205, |
| "kl": 0.028873553121229634, |
| "learning_rate": 3.5684210526315792e-06, |
| "loss": 0.0001, |
| "step": 322 |
| }, |
| { |
| "clip_ratio": 0.00042971508810296655, |
| "epoch": 0.04410158383397051, |
| "grad_norm": 0.0870831087231636, |
| "kl": 0.0294377136597177, |
| "learning_rate": 3.5631578947368424e-06, |
| "loss": -0.0006, |
| "step": 323 |
| }, |
| { |
| "clip_ratio": 0.0004036639875266701, |
| "epoch": 0.04423812124522119, |
| "grad_norm": 0.08122850209474564, |
| "kl": 0.029522280019591562, |
| "learning_rate": 3.5578947368421056e-06, |
| "loss": -0.0011, |
| "step": 324 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.96875, |
| "epoch": 0.044374658656471874, |
| "grad_norm": 0.08125916123390198, |
| "kl": 0.02477030295995064, |
| "learning_rate": 3.5526315789473687e-06, |
| "loss": 0.0002, |
| "num_tokens": 904832.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 325 |
| }, |
| { |
| "clip_ratio": 0.0003188775444868952, |
| "epoch": 0.04451119606772255, |
| "grad_norm": 0.08092335611581802, |
| "kl": 0.025089218630455434, |
| "learning_rate": 3.5473684210526323e-06, |
| "loss": 0.0002, |
| "step": 326 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04464773347897324, |
| "grad_norm": 0.07883549481630325, |
| "kl": 0.024627194390632212, |
| "learning_rate": 3.542105263157895e-06, |
| "loss": -0.0005, |
| "step": 327 |
| }, |
| { |
| "clip_ratio": 0.00020032051543239504, |
| "epoch": 0.04478427089022392, |
| "grad_norm": 0.07583944499492645, |
| "kl": 0.024948682344984263, |
| "learning_rate": 3.536842105263158e-06, |
| "loss": -0.001, |
| "step": 328 |
| }, |
| { |
| "clip_ratio": 0.0008947963651735336, |
| "completion_length": 113.46875, |
| "epoch": 0.0449208083014746, |
| "grad_norm": 0.09001241624355316, |
| "kl": 0.023796884925104678, |
| "learning_rate": 3.5315789473684214e-06, |
| "loss": 0.0, |
| "num_tokens": 915583.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 329 |
| }, |
| { |
| "clip_ratio": 0.00023854961909819394, |
| "epoch": 0.04505734571272529, |
| "grad_norm": 0.09002924710512161, |
| "kl": 0.023522302479250357, |
| "learning_rate": 3.5263157894736846e-06, |
| "loss": -0.0003, |
| "step": 330 |
| }, |
| { |
| "clip_ratio": 0.0005853904294781387, |
| "epoch": 0.04519388312397597, |
| "grad_norm": 0.0916908010840416, |
| "kl": 0.023461372969904914, |
| "learning_rate": 3.5210526315789477e-06, |
| "loss": -0.0006, |
| "step": 331 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04533042053522665, |
| "grad_norm": 0.07961533963680267, |
| "kl": 0.023377142468234524, |
| "learning_rate": 3.515789473684211e-06, |
| "loss": -0.0014, |
| "step": 332 |
| }, |
| { |
| "clip_ratio": 0.00102028384571895, |
| "completion_length": 118.96875, |
| "epoch": 0.045466957946477336, |
| "grad_norm": 0.06798560172319412, |
| "kl": 0.023123418592149392, |
| "learning_rate": 3.510526315789474e-06, |
| "loss": 0.0004, |
| "num_tokens": 926398.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 333 |
| }, |
| { |
| "clip_ratio": 0.001159263076260686, |
| "epoch": 0.045603495357728015, |
| "grad_norm": 0.10703027248382568, |
| "kl": 0.02294311163132079, |
| "learning_rate": 3.505263157894737e-06, |
| "loss": 0.0001, |
| "step": 334 |
| }, |
| { |
| "clip_ratio": 0.00023148147738538682, |
| "epoch": 0.0457400327689787, |
| "grad_norm": 0.07149892300367355, |
| "kl": 0.024184314184822142, |
| "learning_rate": 3.5e-06, |
| "loss": 0.0001, |
| "step": 335 |
| }, |
| { |
| "clip_ratio": 0.0004500129434745759, |
| "epoch": 0.045876570180229385, |
| "grad_norm": 0.0723360925912857, |
| "kl": 0.02370249154046178, |
| "learning_rate": 3.4947368421052635e-06, |
| "loss": -0.0003, |
| "step": 336 |
| }, |
| { |
| "clip_ratio": 0.00016276042151730508, |
| "completion_length": 137.6875, |
| "epoch": 0.046013107591480064, |
| "grad_norm": 0.08149459958076477, |
| "kl": 0.020294574671424925, |
| "learning_rate": 3.4894736842105263e-06, |
| "loss": 0.0001, |
| "num_tokens": 937840.0, |
| "reward": 0.578125, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.578125, |
| "step": 337 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04614964500273075, |
| "grad_norm": 0.07450702041387558, |
| "kl": 0.020218727935571223, |
| "learning_rate": 3.48421052631579e-06, |
| "loss": -0.0, |
| "step": 338 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04628618241398143, |
| "grad_norm": 0.0721743181347847, |
| "kl": 0.02070053394709248, |
| "learning_rate": 3.478947368421053e-06, |
| "loss": -0.0002, |
| "step": 339 |
| }, |
| { |
| "clip_ratio": 0.0008059149840846658, |
| "epoch": 0.04642271982523211, |
| "grad_norm": 0.06328802555799484, |
| "kl": 0.02160324275610037, |
| "learning_rate": 3.473684210526316e-06, |
| "loss": -0.0007, |
| "step": 340 |
| }, |
| { |
| "clip_ratio": 0.000163612567121163, |
| "completion_length": 139.5, |
| "epoch": 0.0465592572364828, |
| "grad_norm": 0.07692926377058029, |
| "kl": 0.022069479804486036, |
| "learning_rate": 3.4684210526315794e-06, |
| "loss": -0.0001, |
| "num_tokens": 949784.0, |
| "reward": 0.4937500059604645, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.4937500059604645, |
| "step": 341 |
| }, |
| { |
| "clip_ratio": 0.0006625788664678112, |
| "epoch": 0.04669579464773348, |
| "grad_norm": 0.07928881049156189, |
| "kl": 0.021967595210298896, |
| "learning_rate": 3.463157894736842e-06, |
| "loss": -0.0002, |
| "step": 342 |
| }, |
| { |
| "clip_ratio": 0.00040216218621935695, |
| "epoch": 0.04683233205898416, |
| "grad_norm": 0.07507841289043427, |
| "kl": 0.02228320448193699, |
| "learning_rate": 3.4578947368421057e-06, |
| "loss": -0.0007, |
| "step": 343 |
| }, |
| { |
| "clip_ratio": 0.000163612567121163, |
| "epoch": 0.04696886947023485, |
| "grad_norm": 0.06530670821666718, |
| "kl": 0.023109311325242743, |
| "learning_rate": 3.4526315789473684e-06, |
| "loss": -0.001, |
| "step": 344 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.21875, |
| "epoch": 0.047105406881485526, |
| "grad_norm": 0.058630239218473434, |
| "kl": 0.027992002695100382, |
| "learning_rate": 3.447368421052632e-06, |
| "loss": 0.0004, |
| "num_tokens": 960135.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 345 |
| }, |
| { |
| "clip_ratio": 0.0006200397037900984, |
| "epoch": 0.04724194429273621, |
| "grad_norm": 0.058566391468048096, |
| "kl": 0.02831086376681924, |
| "learning_rate": 3.4421052631578947e-06, |
| "loss": 0.0003, |
| "step": 346 |
| }, |
| { |
| "clip_ratio": 0.000590555282542482, |
| "epoch": 0.04737848170398689, |
| "grad_norm": 0.05969183146953583, |
| "kl": 0.028586817148607224, |
| "learning_rate": 3.4368421052631583e-06, |
| "loss": 0.0, |
| "step": 347 |
| }, |
| { |
| "clip_ratio": 0.00046267209108918905, |
| "epoch": 0.047515019115237575, |
| "grad_norm": 0.05286189541220665, |
| "kl": 0.02830476386588998, |
| "learning_rate": 3.4315789473684215e-06, |
| "loss": -0.0003, |
| "step": 348 |
| }, |
| { |
| "clip_ratio": 0.00027901786961592734, |
| "completion_length": 126.96875, |
| "epoch": 0.04765155652648826, |
| "grad_norm": 0.08239299803972244, |
| "kl": 0.02313562305062078, |
| "learning_rate": 3.4263157894736842e-06, |
| "loss": 0.0003, |
| "num_tokens": 971494.0, |
| "reward": 0.35312503576278687, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.35312503576278687, |
| "step": 349 |
| }, |
| { |
| "clip_ratio": 0.0004470286221476272, |
| "epoch": 0.04778809393773894, |
| "grad_norm": 0.08283380419015884, |
| "kl": 0.02339187340112403, |
| "learning_rate": 3.421052631578948e-06, |
| "loss": 0.0, |
| "step": 350 |
| }, |
| { |
| "clip_ratio": 0.000615039374679327, |
| "epoch": 0.047924631348989624, |
| "grad_norm": 0.07904310524463654, |
| "kl": 0.02403911092551425, |
| "learning_rate": 3.4157894736842106e-06, |
| "loss": -0.0004, |
| "step": 351 |
| }, |
| { |
| "clip_ratio": 0.0004470286221476272, |
| "epoch": 0.0480611687602403, |
| "grad_norm": 0.0709666758775711, |
| "kl": 0.024250175367342308, |
| "learning_rate": 3.410526315789474e-06, |
| "loss": -0.0009, |
| "step": 352 |
| }, |
| { |
| "clip_ratio": 0.00040849673678167164, |
| "completion_length": 135.0625, |
| "epoch": 0.04819770617149099, |
| "grad_norm": 0.09687653183937073, |
| "kl": 0.023374153446638957, |
| "learning_rate": 3.405263157894737e-06, |
| "loss": 0.0002, |
| "num_tokens": 983216.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 353 |
| }, |
| { |
| "clip_ratio": 0.0009377389069413766, |
| "epoch": 0.04833424358274167, |
| "grad_norm": 0.09615480154752731, |
| "kl": 0.02371872030198574, |
| "learning_rate": 3.4000000000000005e-06, |
| "loss": 0.0002, |
| "step": 354 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04847078099399235, |
| "grad_norm": 0.09314487129449844, |
| "kl": 0.023429605411365628, |
| "learning_rate": 3.3947368421052636e-06, |
| "loss": -0.0006, |
| "step": 355 |
| }, |
| { |
| "clip_ratio": 0.000633919466054067, |
| "epoch": 0.04860731840524304, |
| "grad_norm": 0.08851853013038635, |
| "kl": 0.023448748368537053, |
| "learning_rate": 3.3894736842105264e-06, |
| "loss": -0.0014, |
| "step": 356 |
| }, |
| { |
| "clip_ratio": 0.00020424836839083582, |
| "completion_length": 133.375, |
| "epoch": 0.04874385581649372, |
| "grad_norm": 0.04887694865465164, |
| "kl": 0.02090101721114479, |
| "learning_rate": 3.38421052631579e-06, |
| "loss": 0.0004, |
| "num_tokens": 994928.0, |
| "reward": 0.4937500059604645, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.4937500059604645, |
| "step": 357 |
| }, |
| { |
| "clip_ratio": 0.0004392107803141698, |
| "epoch": 0.0488803932277444, |
| "grad_norm": 0.047825887799263, |
| "kl": 0.021263578528305516, |
| "learning_rate": 3.3789473684210527e-06, |
| "loss": 0.0001, |
| "step": 358 |
| }, |
| { |
| "clip_ratio": 0.00020424836839083582, |
| "epoch": 0.049016930638995086, |
| "grad_norm": 0.04860318452119827, |
| "kl": 0.021143519785255194, |
| "learning_rate": 3.3736842105263163e-06, |
| "loss": 0.0001, |
| "step": 359 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.049153468050245765, |
| "grad_norm": 0.046374525874853134, |
| "kl": 0.020965132425772026, |
| "learning_rate": 3.368421052631579e-06, |
| "loss": -0.0001, |
| "step": 360 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 123.90625, |
| "epoch": 0.04929000546149645, |
| "grad_norm": 0.0487712062895298, |
| "kl": 0.02792163222329691, |
| "learning_rate": 3.3631578947368426e-06, |
| "loss": 0.0002, |
| "num_tokens": 1006129.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 361 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.049426542872747135, |
| "grad_norm": 0.04795907065272331, |
| "kl": 0.027323424263158813, |
| "learning_rate": 3.3578947368421054e-06, |
| "loss": 0.0003, |
| "step": 362 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.049563080283997814, |
| "grad_norm": 0.047760266810655594, |
| "kl": 0.02758122145314701, |
| "learning_rate": 3.3526315789473685e-06, |
| "loss": 0.0001, |
| "step": 363 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0496996176952485, |
| "grad_norm": 0.04801912233233452, |
| "kl": 0.0270579831849318, |
| "learning_rate": 3.347368421052632e-06, |
| "loss": -0.0001, |
| "step": 364 |
| }, |
| { |
| "clip_ratio": 0.0001849112450145185, |
| "completion_length": 120.8125, |
| "epoch": 0.04983615510649918, |
| "grad_norm": 0.05475917458534241, |
| "kl": 0.020006455102702603, |
| "learning_rate": 3.342105263157895e-06, |
| "loss": 0.0001, |
| "num_tokens": 1016995.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 365 |
| }, |
| { |
| "clip_ratio": 0.0001849112450145185, |
| "epoch": 0.04997269251774986, |
| "grad_norm": 0.05398503318428993, |
| "kl": 0.01969250911497511, |
| "learning_rate": 3.3368421052631584e-06, |
| "loss": 0.0001, |
| "step": 366 |
| }, |
| { |
| "clip_ratio": 0.0002460629912093282, |
| "epoch": 0.05010922992900055, |
| "grad_norm": 0.053278569132089615, |
| "kl": 0.01969363953685388, |
| "learning_rate": 3.331578947368421e-06, |
| "loss": -0.0002, |
| "step": 367 |
| }, |
| { |
| "clip_ratio": 0.0002460629912093282, |
| "epoch": 0.05024576734025123, |
| "grad_norm": 0.05154409632086754, |
| "kl": 0.01971426818636246, |
| "learning_rate": 3.3263157894736848e-06, |
| "loss": -0.0004, |
| "step": 368 |
| }, |
| { |
| "clip_ratio": 0.00048449612222611904, |
| "completion_length": 127.625, |
| "epoch": 0.05038230475150191, |
| "grad_norm": 0.08452387899160385, |
| "kl": 0.020907113037537783, |
| "learning_rate": 3.3210526315789475e-06, |
| "loss": 0.0004, |
| "num_tokens": 1028511.0, |
| "reward": 0.49375003576278687, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.49375003576278687, |
| "step": 369 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0505188421627526, |
| "grad_norm": 0.08542630821466446, |
| "kl": 0.020866161328740418, |
| "learning_rate": 3.3157894736842107e-06, |
| "loss": 0.0002, |
| "step": 370 |
| }, |
| { |
| "clip_ratio": 0.0003730945463757962, |
| "epoch": 0.050655379574003276, |
| "grad_norm": 0.08319123089313507, |
| "kl": 0.021059497754322365, |
| "learning_rate": 3.310526315789474e-06, |
| "loss": -0.0004, |
| "step": 371 |
| }, |
| { |
| "clip_ratio": 0.0005486206209752709, |
| "epoch": 0.05079191698525396, |
| "grad_norm": 0.08353553712368011, |
| "kl": 0.021253676881315187, |
| "learning_rate": 3.305263157894737e-06, |
| "loss": -0.001, |
| "step": 372 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.03125, |
| "epoch": 0.05092845439650464, |
| "grad_norm": 0.08390096575021744, |
| "kl": 0.022265749430516735, |
| "learning_rate": 3.3000000000000006e-06, |
| "loss": 0.0001, |
| "num_tokens": 1039464.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 373 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.051064991807755325, |
| "grad_norm": 0.08395753800868988, |
| "kl": 0.022902959695784375, |
| "learning_rate": 3.2947368421052633e-06, |
| "loss": -0.0, |
| "step": 374 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05120152921900601, |
| "grad_norm": 0.07426461577415466, |
| "kl": 0.02265952795278281, |
| "learning_rate": 3.289473684210527e-06, |
| "loss": -0.0004, |
| "step": 375 |
| }, |
| { |
| "clip_ratio": 0.00038479387876577675, |
| "epoch": 0.05133806663025669, |
| "grad_norm": 0.0708777979016304, |
| "kl": 0.02249322272837162, |
| "learning_rate": 3.2842105263157897e-06, |
| "loss": -0.0009, |
| "step": 376 |
| }, |
| { |
| "clip_ratio": 9.61538462433964e-05, |
| "completion_length": 153.9375, |
| "epoch": 0.051474604041507374, |
| "grad_norm": 0.08031871914863586, |
| "kl": 0.02048320347967092, |
| "learning_rate": 3.278947368421053e-06, |
| "loss": 0.0, |
| "num_tokens": 1052498.0, |
| "reward": 0.49375003576278687, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.49375003576278687, |
| "step": 377 |
| }, |
| { |
| "clip_ratio": 0.00018221575010102242, |
| "epoch": 0.05161114145275805, |
| "grad_norm": 0.07619955390691757, |
| "kl": 0.020257426964235492, |
| "learning_rate": 3.273684210526316e-06, |
| "loss": -0.0001, |
| "step": 378 |
| }, |
| { |
| "clip_ratio": 0.0002741228090599179, |
| "epoch": 0.05174767886400874, |
| "grad_norm": 0.07639969140291214, |
| "kl": 0.02037222741637379, |
| "learning_rate": 3.268421052631579e-06, |
| "loss": -0.0003, |
| "step": 379 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05188421627525942, |
| "grad_norm": 0.06297897547483444, |
| "kl": 0.021020412095822394, |
| "learning_rate": 3.2631578947368423e-06, |
| "loss": -0.0007, |
| "step": 380 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 139.09375, |
| "epoch": 0.0520207536865101, |
| "grad_norm": 0.041881829500198364, |
| "kl": 0.022837618802441284, |
| "learning_rate": 3.2578947368421055e-06, |
| "loss": 0.0002, |
| "num_tokens": 1063853.0, |
| "reward": 0.71875, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.71875, |
| "step": 381 |
| }, |
| { |
| "clip_ratio": 0.00014810427092015743, |
| "epoch": 0.05215729109776079, |
| "grad_norm": 0.04161300137639046, |
| "kl": 0.022631432017078623, |
| "learning_rate": 3.252631578947369e-06, |
| "loss": 0.0001, |
| "step": 382 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05229382850901147, |
| "grad_norm": 0.04173358157277107, |
| "kl": 0.023325582704273984, |
| "learning_rate": 3.247368421052632e-06, |
| "loss": 0.0, |
| "step": 383 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05243036592026215, |
| "grad_norm": 0.03905336186289787, |
| "kl": 0.022985702089499682, |
| "learning_rate": 3.2421052631578945e-06, |
| "loss": -0.0002, |
| "step": 384 |
| }, |
| { |
| "clip_ratio": 0.00015394088404718786, |
| "completion_length": 110.125, |
| "epoch": 0.052566903331512836, |
| "grad_norm": 0.07995563000440598, |
| "kl": 0.020230403955793008, |
| "learning_rate": 3.236842105263158e-06, |
| "loss": 0.0002, |
| "num_tokens": 1074037.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 385 |
| }, |
| { |
| "clip_ratio": 0.00015394088404718786, |
| "epoch": 0.052703440742763515, |
| "grad_norm": 0.08091157674789429, |
| "kl": 0.02018303626391571, |
| "learning_rate": 3.2315789473684213e-06, |
| "loss": -0.0, |
| "step": 386 |
| }, |
| { |
| "clip_ratio": 0.00015394088404718786, |
| "epoch": 0.0528399781540142, |
| "grad_norm": 0.0798361599445343, |
| "kl": 0.020285903723561205, |
| "learning_rate": 3.2263157894736845e-06, |
| "loss": -0.0005, |
| "step": 387 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.052976515565264885, |
| "grad_norm": 0.07278761267662048, |
| "kl": 0.02051667576597538, |
| "learning_rate": 3.2210526315789476e-06, |
| "loss": -0.0011, |
| "step": 388 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.21875, |
| "epoch": 0.053113052976515564, |
| "grad_norm": 0.07617516070604324, |
| "kl": 0.028143407515017316, |
| "learning_rate": 3.215789473684211e-06, |
| "loss": 0.0003, |
| "num_tokens": 1084844.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 389 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05324959038776625, |
| "grad_norm": 0.07584980130195618, |
| "kl": 0.02738626574864611, |
| "learning_rate": 3.210526315789474e-06, |
| "loss": 0.0003, |
| "step": 390 |
| }, |
| { |
| "clip_ratio": 0.0005663055635523051, |
| "epoch": 0.05338612779901693, |
| "grad_norm": 0.07561825215816498, |
| "kl": 0.027368494658730924, |
| "learning_rate": 3.2052631578947367e-06, |
| "loss": -0.0001, |
| "step": 391 |
| }, |
| { |
| "clip_ratio": 0.0005663055635523051, |
| "epoch": 0.05352266521026761, |
| "grad_norm": 0.06851368397474289, |
| "kl": 0.027929118426982313, |
| "learning_rate": 3.2000000000000003e-06, |
| "loss": -0.0006, |
| "step": 392 |
| }, |
| { |
| "clip_ratio": 0.00021853146608918905, |
| "completion_length": 120.71875, |
| "epoch": 0.0536592026215183, |
| "grad_norm": 0.058026522397994995, |
| "kl": 0.025286480551585555, |
| "learning_rate": 3.1947368421052634e-06, |
| "loss": 0.0001, |
| "num_tokens": 1096015.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 393 |
| }, |
| { |
| "clip_ratio": 0.00021853146608918905, |
| "epoch": 0.05379574003276898, |
| "grad_norm": 0.05753491073846817, |
| "kl": 0.0260978079168126, |
| "learning_rate": 3.1894736842105266e-06, |
| "loss": 0.0, |
| "step": 394 |
| }, |
| { |
| "clip_ratio": 0.00021853146608918905, |
| "epoch": 0.05393227744401966, |
| "grad_norm": 0.05408268794417381, |
| "kl": 0.026300053868908435, |
| "learning_rate": 3.1842105263157898e-06, |
| "loss": -0.0002, |
| "step": 395 |
| }, |
| { |
| "clip_ratio": 0.00020424836839083582, |
| "epoch": 0.05406881485527035, |
| "grad_norm": 0.04638107120990753, |
| "kl": 0.02564926413469948, |
| "learning_rate": 3.178947368421053e-06, |
| "loss": -0.0005, |
| "step": 396 |
| }, |
| { |
| "clip_ratio": 0.00035511364694684744, |
| "completion_length": 102.875, |
| "epoch": 0.054205352266521026, |
| "grad_norm": 0.08631628006696701, |
| "kl": 0.02725237447884865, |
| "learning_rate": 3.173684210526316e-06, |
| "loss": 0.0004, |
| "num_tokens": 1106459.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 397 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05434188967777171, |
| "grad_norm": 0.0849158763885498, |
| "kl": 0.02785361756104976, |
| "learning_rate": 3.168421052631579e-06, |
| "loss": 0.0001, |
| "step": 398 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05447842708902239, |
| "grad_norm": 0.08003275096416473, |
| "kl": 0.027714760246453807, |
| "learning_rate": 3.1631578947368424e-06, |
| "loss": -0.0005, |
| "step": 399 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.054614964500273075, |
| "grad_norm": 0.06917204707860947, |
| "kl": 0.027757962932810187, |
| "learning_rate": 3.157894736842105e-06, |
| "loss": -0.0008, |
| "step": 400 |
| }, |
| { |
| "clip_ratio": 0.00020833333837799728, |
| "completion_length": 114.53125, |
| "epoch": 0.05475150191152376, |
| "grad_norm": 0.054499052464962006, |
| "kl": 0.02256640234554652, |
| "learning_rate": 3.1526315789473688e-06, |
| "loss": 0.0001, |
| "num_tokens": 1117220.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 401 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05488803932277444, |
| "grad_norm": 0.054717376828193665, |
| "kl": 0.022691508871503174, |
| "learning_rate": 3.147368421052632e-06, |
| "loss": -0.0, |
| "step": 402 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.055024576734025124, |
| "grad_norm": 0.053422003984451294, |
| "kl": 0.023062629101332277, |
| "learning_rate": 3.142105263157895e-06, |
| "loss": -0.0002, |
| "step": 403 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0551611141452758, |
| "grad_norm": 0.05216868966817856, |
| "kl": 0.022557226198841818, |
| "learning_rate": 3.1368421052631582e-06, |
| "loss": -0.0006, |
| "step": 404 |
| }, |
| { |
| "completion_length": 115.21875, |
| "epoch": 0.05529765155652649, |
| "grad_norm": 0.0, |
| "learning_rate": 3.131578947368421e-06, |
| "loss": 0.0, |
| "num_tokens": 1128051.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.0, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.05543418896777717, |
| "grad_norm": 0.0, |
| "learning_rate": 3.1263157894736846e-06, |
| "loss": 0.0, |
| "step": 406 |
| }, |
| { |
| "epoch": 0.05557072637902785, |
| "grad_norm": 0.0, |
| "learning_rate": 3.1210526315789473e-06, |
| "loss": 0.0, |
| "step": 407 |
| }, |
| { |
| "epoch": 0.05570726379027854, |
| "grad_norm": 0.0, |
| "learning_rate": 3.115789473684211e-06, |
| "loss": 0.0, |
| "step": 408 |
| }, |
| { |
| "clip_ratio": 0.0004936508776154369, |
| "completion_length": 120.84375, |
| "epoch": 0.05584380120152922, |
| "grad_norm": 0.09440712630748749, |
| "kl": 0.029624507413245738, |
| "learning_rate": 3.1105263157894736e-06, |
| "loss": 0.0005, |
| "num_tokens": 1139234.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 409 |
| }, |
| { |
| "clip_ratio": 0.0004337021673563868, |
| "epoch": 0.0559803386127799, |
| "grad_norm": 0.09525737166404724, |
| "kl": 0.02997314796084538, |
| "learning_rate": 3.1052631578947372e-06, |
| "loss": 0.0003, |
| "step": 410 |
| }, |
| { |
| "clip_ratio": 0.000668147680698894, |
| "epoch": 0.056116876024030586, |
| "grad_norm": 0.09334629029035568, |
| "kl": 0.029849036916857585, |
| "learning_rate": 3.1000000000000004e-06, |
| "loss": -0.0002, |
| "step": 411 |
| }, |
| { |
| "clip_ratio": 0.0006625162932323292, |
| "epoch": 0.056253413435281265, |
| "grad_norm": 0.09088991582393646, |
| "kl": 0.029855199943995103, |
| "learning_rate": 3.094736842105263e-06, |
| "loss": -0.0009, |
| "step": 412 |
| }, |
| { |
| "clip_ratio": 0.00026709403027780354, |
| "completion_length": 121.4375, |
| "epoch": 0.05638995084653195, |
| "grad_norm": 0.08999986946582794, |
| "kl": 0.0211114841804374, |
| "learning_rate": 3.0894736842105267e-06, |
| "loss": 0.0001, |
| "num_tokens": 1150204.0, |
| "reward": 0.40937501192092896, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.40937501192092896, |
| "step": 413 |
| }, |
| { |
| "clip_ratio": 0.00044984256965108216, |
| "epoch": 0.056526488257782635, |
| "grad_norm": 0.0902203768491745, |
| "kl": 0.021156518894713372, |
| "learning_rate": 3.0842105263157895e-06, |
| "loss": -0.0, |
| "step": 414 |
| }, |
| { |
| "clip_ratio": 0.000660560792312026, |
| "epoch": 0.056663025669033314, |
| "grad_norm": 0.08662670105695724, |
| "kl": 0.021238182176603004, |
| "learning_rate": 3.078947368421053e-06, |
| "loss": -0.0004, |
| "step": 415 |
| }, |
| { |
| "clip_ratio": 0.0004912797885481268, |
| "epoch": 0.056799563080284, |
| "grad_norm": 0.08425270766019821, |
| "kl": 0.021560683904681355, |
| "learning_rate": 3.0736842105263158e-06, |
| "loss": -0.0013, |
| "step": 416 |
| }, |
| { |
| "clip_ratio": 0.000488749414216727, |
| "completion_length": 121.5, |
| "epoch": 0.05693610049153468, |
| "grad_norm": 0.07870546728372574, |
| "kl": 0.02753204884356819, |
| "learning_rate": 3.0684210526315794e-06, |
| "loss": 0.0005, |
| "num_tokens": 1161660.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 417 |
| }, |
| { |
| "clip_ratio": 0.00045018277887720615, |
| "epoch": 0.05707263790278536, |
| "grad_norm": 0.0780278816819191, |
| "kl": 0.02779330688645132, |
| "learning_rate": 3.0631578947368425e-06, |
| "loss": 0.0004, |
| "step": 418 |
| }, |
| { |
| "clip_ratio": 0.0009154355502687395, |
| "epoch": 0.05720917531403605, |
| "grad_norm": 0.07824473083019257, |
| "kl": 0.027306273754220456, |
| "learning_rate": 3.0578947368421053e-06, |
| "loss": 0.0, |
| "step": 419 |
| }, |
| { |
| "clip_ratio": 0.0002736291062319651, |
| "epoch": 0.05734571272528673, |
| "grad_norm": 0.0745638981461525, |
| "kl": 0.027793304558144882, |
| "learning_rate": 3.052631578947369e-06, |
| "loss": -0.0006, |
| "step": 420 |
| }, |
| { |
| "clip_ratio": 0.0003396739193703979, |
| "completion_length": 103.65625, |
| "epoch": 0.05748225013653741, |
| "grad_norm": 0.042599353939294815, |
| "kl": 0.028534257464343682, |
| "learning_rate": 3.0473684210526316e-06, |
| "loss": 0.0003, |
| "num_tokens": 1171885.0, |
| "reward": 0.578125, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.578125, |
| "step": 421 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0576187875477881, |
| "grad_norm": 0.042493246495723724, |
| "kl": 0.028584119427250698, |
| "learning_rate": 3.042105263157895e-06, |
| "loss": 0.0002, |
| "step": 422 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.057755324959038776, |
| "grad_norm": 0.042571090161800385, |
| "kl": 0.02841770788654685, |
| "learning_rate": 3.036842105263158e-06, |
| "loss": 0.0001, |
| "step": 423 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05789186237028946, |
| "grad_norm": 0.042970895767211914, |
| "kl": 0.028437837143428624, |
| "learning_rate": 3.0315789473684215e-06, |
| "loss": -0.0001, |
| "step": 424 |
| }, |
| { |
| "clip_ratio": 0.0006481033633463085, |
| "completion_length": 119.5, |
| "epoch": 0.05802839978154014, |
| "grad_norm": 0.08428654819726944, |
| "kl": 0.03005171535187401, |
| "learning_rate": 3.0263157894736843e-06, |
| "loss": 0.0003, |
| "num_tokens": 1182569.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 425 |
| }, |
| { |
| "clip_ratio": 0.00028153153834864497, |
| "epoch": 0.058164937192790825, |
| "grad_norm": 0.08224932104349136, |
| "kl": 0.029877902969019488, |
| "learning_rate": 3.0210526315789474e-06, |
| "loss": 0.0, |
| "step": 426 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05830147460404151, |
| "grad_norm": 0.0772838145494461, |
| "kl": 0.02998991316417232, |
| "learning_rate": 3.015789473684211e-06, |
| "loss": -0.0003, |
| "step": 427 |
| }, |
| { |
| "clip_ratio": 0.00028153153834864497, |
| "epoch": 0.05843801201529219, |
| "grad_norm": 0.06520218402147293, |
| "kl": 0.029636650171596557, |
| "learning_rate": 3.0105263157894737e-06, |
| "loss": -0.0008, |
| "step": 428 |
| }, |
| { |
| "clip_ratio": 0.00034340660204179585, |
| "completion_length": 110.90625, |
| "epoch": 0.058574549426542874, |
| "grad_norm": 0.05254102498292923, |
| "kl": 0.02795352225075476, |
| "learning_rate": 3.0052631578947373e-06, |
| "loss": 0.0004, |
| "num_tokens": 1192970.0, |
| "reward": 0.859375, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.859375, |
| "step": 429 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05871108683779355, |
| "grad_norm": 0.05203205719590187, |
| "kl": 0.027702660532668233, |
| "learning_rate": 3e-06, |
| "loss": 0.0003, |
| "step": 430 |
| }, |
| { |
| "clip_ratio": 0.00034340660204179585, |
| "epoch": 0.05884762424904424, |
| "grad_norm": 0.050078123807907104, |
| "kl": 0.027325164119247347, |
| "learning_rate": 2.9947368421052637e-06, |
| "loss": 0.0001, |
| "step": 431 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05898416166029492, |
| "grad_norm": 0.04930827394127846, |
| "kl": 0.028029807755956426, |
| "learning_rate": 2.9894736842105264e-06, |
| "loss": -0.0001, |
| "step": 432 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.625, |
| "epoch": 0.0591206990715456, |
| "grad_norm": 0.11645514518022537, |
| "kl": 0.029349218850256875, |
| "learning_rate": 2.9842105263157896e-06, |
| "loss": 0.0004, |
| "num_tokens": 1203310.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 433 |
| }, |
| { |
| "clip_ratio": 0.0003522308688843623, |
| "epoch": 0.05925723648279629, |
| "grad_norm": 0.1155751571059227, |
| "kl": 0.029482927988283336, |
| "learning_rate": 2.9789473684210527e-06, |
| "loss": -0.0003, |
| "step": 434 |
| }, |
| { |
| "clip_ratio": 0.0015326489083236083, |
| "epoch": 0.05939377389404697, |
| "grad_norm": 0.1054697334766388, |
| "kl": 0.02898168022511527, |
| "learning_rate": 2.973684210526316e-06, |
| "loss": -0.0008, |
| "step": 435 |
| }, |
| { |
| "clip_ratio": 0.0028316913376329467, |
| "epoch": 0.05953031130529765, |
| "grad_norm": 0.08968142420053482, |
| "kl": 0.02963988418923691, |
| "learning_rate": 2.9684210526315795e-06, |
| "loss": -0.0016, |
| "step": 436 |
| }, |
| { |
| "clip_ratio": 0.00034131205757148564, |
| "completion_length": 124.125, |
| "epoch": 0.059666848716548336, |
| "grad_norm": 0.11992131173610687, |
| "kl": 0.02362085011554882, |
| "learning_rate": 2.9631578947368422e-06, |
| "loss": 0.0004, |
| "num_tokens": 1215018.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.35490378737449646, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 437 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.059803386127799014, |
| "grad_norm": 0.10817261785268784, |
| "kl": 0.023265416646609083, |
| "learning_rate": 2.957894736842106e-06, |
| "loss": -0.0002, |
| "step": 438 |
| }, |
| { |
| "clip_ratio": 0.00013297871919348836, |
| "epoch": 0.0599399235390497, |
| "grad_norm": 0.10226042568683624, |
| "kl": 0.023465301826945506, |
| "learning_rate": 2.9526315789473685e-06, |
| "loss": -0.0006, |
| "step": 439 |
| }, |
| { |
| "clip_ratio": 0.0010071903670905158, |
| "epoch": 0.060076460950300385, |
| "grad_norm": 0.09243166446685791, |
| "kl": 0.02359252143651247, |
| "learning_rate": 2.9473684210526317e-06, |
| "loss": -0.0013, |
| "step": 440 |
| }, |
| { |
| "clip_ratio": 0.00022007041843608022, |
| "completion_length": 105.8125, |
| "epoch": 0.060212998361551064, |
| "grad_norm": 0.09433529525995255, |
| "kl": 0.02786622042185627, |
| "learning_rate": 2.942105263157895e-06, |
| "loss": 0.0003, |
| "num_tokens": 1225400.0, |
| "reward": 0.71875, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.71875, |
| "step": 441 |
| }, |
| { |
| "clip_ratio": 0.0003289473825134337, |
| "epoch": 0.06034953577280175, |
| "grad_norm": 0.09359579533338547, |
| "kl": 0.02748560524196364, |
| "learning_rate": 2.936842105263158e-06, |
| "loss": 0.0002, |
| "step": 442 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06048607318405243, |
| "grad_norm": 0.09082359075546265, |
| "kl": 0.027210222935536876, |
| "learning_rate": 2.931578947368421e-06, |
| "loss": -0.0006, |
| "step": 443 |
| }, |
| { |
| "clip_ratio": 0.00028153153834864497, |
| "epoch": 0.06062261059530311, |
| "grad_norm": 0.08561991155147552, |
| "kl": 0.02759760251501575, |
| "learning_rate": 2.9263157894736844e-06, |
| "loss": -0.0013, |
| "step": 444 |
| }, |
| { |
| "clip_ratio": 0.00017959770048037171, |
| "completion_length": 121.34375, |
| "epoch": 0.0607591480065538, |
| "grad_norm": 0.08825256675481796, |
| "kl": 0.023633092801901512, |
| "learning_rate": 2.921052631578948e-06, |
| "loss": 0.0003, |
| "num_tokens": 1236679.0, |
| "reward": 0.6624999642372131, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.6624999642372131, |
| "step": 445 |
| }, |
| { |
| "clip_ratio": 0.00039966811891645193, |
| "epoch": 0.060895685417804477, |
| "grad_norm": 0.08804938942193985, |
| "kl": 0.02338150296418462, |
| "learning_rate": 2.9157894736842107e-06, |
| "loss": -0.0001, |
| "step": 446 |
| }, |
| { |
| "clip_ratio": 0.0005587452324107289, |
| "epoch": 0.06103222282905516, |
| "grad_norm": 0.08598316460847855, |
| "kl": 0.023485250319936313, |
| "learning_rate": 2.9105263157894743e-06, |
| "loss": -0.0003, |
| "step": 447 |
| }, |
| { |
| "clip_ratio": 0.0005587452324107289, |
| "epoch": 0.06116876024030585, |
| "grad_norm": 0.08363504707813263, |
| "kl": 0.023482873002649285, |
| "learning_rate": 2.905263157894737e-06, |
| "loss": -0.0009, |
| "step": 448 |
| }, |
| { |
| "clip_ratio": 0.000234962411923334, |
| "completion_length": 128.09375, |
| "epoch": 0.061305297651556526, |
| "grad_norm": 0.09320642799139023, |
| "kl": 0.025599762244382873, |
| "learning_rate": 2.9e-06, |
| "loss": 0.0002, |
| "num_tokens": 1247894.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 449 |
| }, |
| { |
| "clip_ratio": 0.000234962411923334, |
| "epoch": 0.06144183506280721, |
| "grad_norm": 0.0868094265460968, |
| "kl": 0.02552194581949152, |
| "learning_rate": 2.8947368421052634e-06, |
| "loss": -0.0001, |
| "step": 450 |
| }, |
| { |
| "clip_ratio": 0.000234962411923334, |
| "epoch": 0.06157837247405789, |
| "grad_norm": 0.07934384793043137, |
| "kl": 0.024943586031440645, |
| "learning_rate": 2.8894736842105265e-06, |
| "loss": -0.0005, |
| "step": 451 |
| }, |
| { |
| "clip_ratio": 0.000234962411923334, |
| "epoch": 0.061714909885308575, |
| "grad_norm": 0.07852542400360107, |
| "kl": 0.024750137235969305, |
| "learning_rate": 2.88421052631579e-06, |
| "loss": -0.0009, |
| "step": 452 |
| }, |
| { |
| "clip_ratio": 0.0010345236223656684, |
| "completion_length": 118.21875, |
| "epoch": 0.06185144729655926, |
| "grad_norm": 0.12858963012695312, |
| "kl": 0.02785053660045378, |
| "learning_rate": 2.878947368421053e-06, |
| "loss": 0.0006, |
| "num_tokens": 1258989.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.40245187282562256, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 453 |
| }, |
| { |
| "clip_ratio": 0.0012061969318892807, |
| "epoch": 0.06198798470780994, |
| "grad_norm": 0.12805882096290588, |
| "kl": 0.02756309215328656, |
| "learning_rate": 2.8736842105263164e-06, |
| "loss": 0.0001, |
| "step": 454 |
| }, |
| { |
| "clip_ratio": 0.0009005931788124144, |
| "epoch": 0.062124522119060624, |
| "grad_norm": 0.12524963915348053, |
| "kl": 0.027533066226169467, |
| "learning_rate": 2.868421052631579e-06, |
| "loss": -0.0012, |
| "step": 455 |
| }, |
| { |
| "clip_ratio": 0.0008774404122959822, |
| "epoch": 0.0622610595303113, |
| "grad_norm": 0.11410316824913025, |
| "kl": 0.028351856861263514, |
| "learning_rate": 2.8631578947368423e-06, |
| "loss": -0.002, |
| "step": 456 |
| }, |
| { |
| "clip_ratio": 0.00044964029802940786, |
| "completion_length": 119.75, |
| "epoch": 0.06239759694156199, |
| "grad_norm": 0.0884980857372284, |
| "kl": 0.02622422287822701, |
| "learning_rate": 2.8578947368421055e-06, |
| "loss": 0.0003, |
| "num_tokens": 1269833.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 457 |
| }, |
| { |
| "clip_ratio": 0.0004271467769285664, |
| "epoch": 0.06253413435281267, |
| "grad_norm": 0.08802725374698639, |
| "kl": 0.0259568098699674, |
| "learning_rate": 2.8526315789473687e-06, |
| "loss": -0.0, |
| "step": 458 |
| }, |
| { |
| "clip_ratio": 0.0007370132370851934, |
| "epoch": 0.06267067176406335, |
| "grad_norm": 0.07317302376031876, |
| "kl": 0.0255898576870095, |
| "learning_rate": 2.847368421052632e-06, |
| "loss": -0.0004, |
| "step": 459 |
| }, |
| { |
| "clip_ratio": 0.0002281021879753098, |
| "epoch": 0.06280720917531403, |
| "grad_norm": 0.07119499146938324, |
| "kl": 0.025962556974263862, |
| "learning_rate": 2.842105263157895e-06, |
| "loss": -0.0009, |
| "step": 460 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.9375, |
| "epoch": 0.06294374658656472, |
| "grad_norm": 0.07267936319112778, |
| "kl": 0.030646027327748016, |
| "learning_rate": 2.8368421052631586e-06, |
| "loss": 0.0002, |
| "num_tokens": 1280727.0, |
| "reward": 0.7750000357627869, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7750000357627869, |
| "step": 461 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0630802839978154, |
| "grad_norm": 0.07068201154470444, |
| "kl": 0.030932883906643838, |
| "learning_rate": 2.8315789473684213e-06, |
| "loss": 0.0001, |
| "step": 462 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06321682140906608, |
| "grad_norm": 0.06675128638744354, |
| "kl": 0.03135033437865786, |
| "learning_rate": 2.826315789473684e-06, |
| "loss": -0.0003, |
| "step": 463 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06335335882031677, |
| "grad_norm": 0.060258980840444565, |
| "kl": 0.031988496135454625, |
| "learning_rate": 2.8210526315789476e-06, |
| "loss": -0.0004, |
| "step": 464 |
| }, |
| { |
| "clip_ratio": 0.0005117313994560391, |
| "completion_length": 122.84375, |
| "epoch": 0.06348989623156745, |
| "grad_norm": 0.07553315907716751, |
| "kl": 0.03090066317236051, |
| "learning_rate": 2.815789473684211e-06, |
| "loss": 0.0003, |
| "num_tokens": 1291590.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 465 |
| }, |
| { |
| "clip_ratio": 0.0006195175519678742, |
| "epoch": 0.06362643364281813, |
| "grad_norm": 0.07488815486431122, |
| "kl": 0.031175631389487535, |
| "learning_rate": 2.810526315789474e-06, |
| "loss": 0.0002, |
| "step": 466 |
| }, |
| { |
| "clip_ratio": 0.00039545909385196865, |
| "epoch": 0.06376297105406882, |
| "grad_norm": 0.07623933255672455, |
| "kl": 0.030218343308661133, |
| "learning_rate": 2.805263157894737e-06, |
| "loss": -0.0003, |
| "step": 467 |
| }, |
| { |
| "clip_ratio": 0.00041963320109061897, |
| "epoch": 0.0638995084653195, |
| "grad_norm": 0.06566479057073593, |
| "kl": 0.030176448373822495, |
| "learning_rate": 2.8000000000000003e-06, |
| "loss": -0.0006, |
| "step": 468 |
| }, |
| { |
| "clip_ratio": 0.000202922077733092, |
| "completion_length": 143.6875, |
| "epoch": 0.06403604587657018, |
| "grad_norm": 0.07517211139202118, |
| "kl": 0.026966167468344793, |
| "learning_rate": 2.7947368421052635e-06, |
| "loss": 0.0001, |
| "num_tokens": 1303640.0, |
| "reward": 0.6906249523162842, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.6906249523162842, |
| "step": 469 |
| }, |
| { |
| "clip_ratio": 0.0011654012050712481, |
| "epoch": 0.06417258328782087, |
| "grad_norm": 0.07574381679296494, |
| "kl": 0.026625617261743173, |
| "learning_rate": 2.789473684210526e-06, |
| "loss": -0.0001, |
| "step": 470 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06430912069907155, |
| "grad_norm": 0.06845681369304657, |
| "kl": 0.027047655457863584, |
| "learning_rate": 2.7842105263157898e-06, |
| "loss": -0.0004, |
| "step": 471 |
| }, |
| { |
| "clip_ratio": 0.0004459502233657986, |
| "epoch": 0.06444565811032223, |
| "grad_norm": 0.06472209841012955, |
| "kl": 0.02748362798593007, |
| "learning_rate": 2.7789473684210525e-06, |
| "loss": -0.0007, |
| "step": 472 |
| }, |
| { |
| "clip_ratio": 0.00017265194037463516, |
| "completion_length": 100.0, |
| "epoch": 0.0645821955215729, |
| "grad_norm": 0.08828121423721313, |
| "kl": 0.029740739322733134, |
| "learning_rate": 2.773684210526316e-06, |
| "loss": 0.0004, |
| "num_tokens": 1314068.0, |
| "reward": 0.859375, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.859375, |
| "step": 473 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0647187329328236, |
| "grad_norm": 0.08898042142391205, |
| "kl": 0.029690030263736844, |
| "learning_rate": 2.7684210526315793e-06, |
| "loss": 0.0001, |
| "step": 474 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06485527034407428, |
| "grad_norm": 0.08371194452047348, |
| "kl": 0.030416715890169144, |
| "learning_rate": 2.7631578947368424e-06, |
| "loss": -0.0003, |
| "step": 475 |
| }, |
| { |
| "clip_ratio": 0.0003927223588107154, |
| "epoch": 0.06499180775532495, |
| "grad_norm": 0.08132834732532501, |
| "kl": 0.03134842653525993, |
| "learning_rate": 2.7578947368421056e-06, |
| "loss": -0.0009, |
| "step": 476 |
| }, |
| { |
| "completion_length": 111.34375, |
| "epoch": 0.06512834516657565, |
| "grad_norm": 0.0, |
| "learning_rate": 2.7526315789473683e-06, |
| "loss": 0.0, |
| "num_tokens": 1325395.0, |
| "reward": 0.887499988079071, |
| "reward_std": 0.0, |
| "rewards/reward_fn": 0.887499988079071, |
| "step": 477 |
| }, |
| { |
| "epoch": 0.06526488257782632, |
| "grad_norm": 0.0, |
| "learning_rate": 2.747368421052632e-06, |
| "loss": 0.0, |
| "step": 478 |
| }, |
| { |
| "epoch": 0.065401419989077, |
| "grad_norm": 0.0, |
| "learning_rate": 2.7421052631578947e-06, |
| "loss": 0.0, |
| "step": 479 |
| }, |
| { |
| "epoch": 0.0655379574003277, |
| "grad_norm": 0.0, |
| "learning_rate": 2.7368421052631583e-06, |
| "loss": 0.0, |
| "step": 480 |
| }, |
| { |
| "clip_ratio": 0.0008182729070540518, |
| "completion_length": 122.4375, |
| "epoch": 0.06567449481157837, |
| "grad_norm": 0.1075565367937088, |
| "kl": 0.03270941000664607, |
| "learning_rate": 2.7315789473684214e-06, |
| "loss": 0.0004, |
| "num_tokens": 1336833.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.2511056959629059, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 481 |
| }, |
| { |
| "clip_ratio": 0.0002460629912093282, |
| "epoch": 0.06581103222282905, |
| "grad_norm": 0.10678515583276749, |
| "kl": 0.03262783802347258, |
| "learning_rate": 2.7263157894736846e-06, |
| "loss": 0.0003, |
| "step": 482 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06594756963407974, |
| "grad_norm": 0.10226035863161087, |
| "kl": 0.03304731094976887, |
| "learning_rate": 2.7210526315789478e-06, |
| "loss": -0.0003, |
| "step": 483 |
| }, |
| { |
| "clip_ratio": 0.00023854961909819394, |
| "epoch": 0.06608410704533042, |
| "grad_norm": 0.09197822958230972, |
| "kl": 0.03369238827144727, |
| "learning_rate": 2.7157894736842105e-06, |
| "loss": -0.001, |
| "step": 484 |
| }, |
| { |
| "clip_ratio": 0.00024801588733680546, |
| "completion_length": 105.90625, |
| "epoch": 0.0662206444565811, |
| "grad_norm": 0.10018602758646011, |
| "kl": 0.03327567869564518, |
| "learning_rate": 2.710526315789474e-06, |
| "loss": 0.0004, |
| "num_tokens": 1346950.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 485 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06635718186783178, |
| "grad_norm": 0.09925264865159988, |
| "kl": 0.032918925397098064, |
| "learning_rate": 2.705263157894737e-06, |
| "loss": 0.0003, |
| "step": 486 |
| }, |
| { |
| "clip_ratio": 0.0005063201388111338, |
| "epoch": 0.06649371927908247, |
| "grad_norm": 0.0975351333618164, |
| "kl": 0.03307101898826659, |
| "learning_rate": 2.7000000000000004e-06, |
| "loss": -0.0003, |
| "step": 487 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06663025669033315, |
| "grad_norm": 0.0953407809138298, |
| "kl": 0.03223849239293486, |
| "learning_rate": 2.694736842105263e-06, |
| "loss": -0.001, |
| "step": 488 |
| }, |
| { |
| "clip_ratio": 0.0003289473825134337, |
| "completion_length": 120.3125, |
| "epoch": 0.06676679410158383, |
| "grad_norm": 0.09660880267620087, |
| "kl": 0.03435020908364095, |
| "learning_rate": 2.6894736842105267e-06, |
| "loss": 0.0004, |
| "num_tokens": 1357580.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 489 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06690333151283452, |
| "grad_norm": 0.09672436863183975, |
| "kl": 0.03443722132942639, |
| "learning_rate": 2.68421052631579e-06, |
| "loss": 0.0002, |
| "step": 490 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0670398689240852, |
| "grad_norm": 0.0930391252040863, |
| "kl": 0.0339292844873853, |
| "learning_rate": 2.6789473684210526e-06, |
| "loss": -0.0002, |
| "step": 491 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06717640633533588, |
| "grad_norm": 0.08958112448453903, |
| "kl": 0.03364733717171475, |
| "learning_rate": 2.6736842105263162e-06, |
| "loss": -0.0008, |
| "step": 492 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.125, |
| "epoch": 0.06731294374658657, |
| "grad_norm": 0.05487014725804329, |
| "kl": 0.0318443511496298, |
| "learning_rate": 2.668421052631579e-06, |
| "loss": 0.0003, |
| "num_tokens": 1368584.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 493 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06744948115783725, |
| "grad_norm": 0.054498445242643356, |
| "kl": 0.0327269172121305, |
| "learning_rate": 2.6631578947368426e-06, |
| "loss": 0.0001, |
| "step": 494 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06758601856908793, |
| "grad_norm": 0.053584616631269455, |
| "kl": 0.03137458971468732, |
| "learning_rate": 2.6578947368421053e-06, |
| "loss": -0.0001, |
| "step": 495 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06772255598033862, |
| "grad_norm": 0.05226173624396324, |
| "kl": 0.03136958571849391, |
| "learning_rate": 2.652631578947369e-06, |
| "loss": -0.0002, |
| "step": 496 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.84375, |
| "epoch": 0.0678590933915893, |
| "grad_norm": 0.10085256397724152, |
| "kl": 0.028638561692787334, |
| "learning_rate": 2.6473684210526316e-06, |
| "loss": 0.0002, |
| "num_tokens": 1379079.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 497 |
| }, |
| { |
| "clip_ratio": 0.000477544468594715, |
| "epoch": 0.06799563080283998, |
| "grad_norm": 0.09922110289335251, |
| "kl": 0.028383127588313073, |
| "learning_rate": 2.6421052631578948e-06, |
| "loss": -0.0001, |
| "step": 498 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06813216821409065, |
| "grad_norm": 0.09748240560293198, |
| "kl": 0.028070454631233588, |
| "learning_rate": 2.6368421052631584e-06, |
| "loss": -0.0007, |
| "step": 499 |
| }, |
| { |
| "clip_ratio": 0.00023148147738538682, |
| "epoch": 0.06826870562534135, |
| "grad_norm": 0.08522222936153412, |
| "kl": 0.027039629960199818, |
| "learning_rate": 2.631578947368421e-06, |
| "loss": -0.0012, |
| "step": 500 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 121.59375, |
| "epoch": 0.06840524303659203, |
| "grad_norm": 0.05568603426218033, |
| "kl": 0.029449922149069607, |
| "learning_rate": 2.6263157894736847e-06, |
| "loss": 0.0003, |
| "num_tokens": 1389594.0, |
| "reward": 0.8875000476837158, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.8875000476837158, |
| "step": 501 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0685417804478427, |
| "grad_norm": 0.05571011081337929, |
| "kl": 0.029838646500138566, |
| "learning_rate": 2.6210526315789474e-06, |
| "loss": 0.0002, |
| "step": 502 |
| }, |
| { |
| "clip_ratio": 0.00016622339899186045, |
| "epoch": 0.0686783178590934, |
| "grad_norm": 0.055335305631160736, |
| "kl": 0.028605365252587944, |
| "learning_rate": 2.615789473684211e-06, |
| "loss": 0.0001, |
| "step": 503 |
| }, |
| { |
| "clip_ratio": 0.0001806358341127634, |
| "epoch": 0.06881485527034407, |
| "grad_norm": 0.053467657417058945, |
| "kl": 0.02818925876636058, |
| "learning_rate": 2.6105263157894738e-06, |
| "loss": -0.0001, |
| "step": 504 |
| }, |
| { |
| "clip_ratio": 0.00041107917786575854, |
| "completion_length": 112.375, |
| "epoch": 0.06895139268159475, |
| "grad_norm": 0.07417155802249908, |
| "kl": 0.023815288412151858, |
| "learning_rate": 2.605263157894737e-06, |
| "loss": 0.0004, |
| "num_tokens": 1399702.0, |
| "reward": 0.859375, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.859375, |
| "step": 505 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06908793009284545, |
| "grad_norm": 0.07561016082763672, |
| "kl": 0.02323581191012636, |
| "learning_rate": 2.6e-06, |
| "loss": 0.0002, |
| "step": 506 |
| }, |
| { |
| "clip_ratio": 0.0004826326767215505, |
| "epoch": 0.06922446750409612, |
| "grad_norm": 0.0658431127667427, |
| "kl": 0.02391452860319987, |
| "learning_rate": 2.5947368421052633e-06, |
| "loss": -0.0002, |
| "step": 507 |
| }, |
| { |
| "clip_ratio": 0.00017959770048037171, |
| "epoch": 0.0693610049153468, |
| "grad_norm": 0.06122962757945061, |
| "kl": 0.023273998143849894, |
| "learning_rate": 2.589473684210527e-06, |
| "loss": -0.0004, |
| "step": 508 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.09375, |
| "epoch": 0.0694975423265975, |
| "grad_norm": 0.08124542981386185, |
| "kl": 0.026170960336457938, |
| "learning_rate": 2.5842105263157896e-06, |
| "loss": 0.0002, |
| "num_tokens": 1410113.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 509 |
| }, |
| { |
| "clip_ratio": 0.0004949534049956128, |
| "epoch": 0.06963407973784817, |
| "grad_norm": 0.07992769777774811, |
| "kl": 0.026557496574241668, |
| "learning_rate": 2.578947368421053e-06, |
| "loss": 0.0, |
| "step": 510 |
| }, |
| { |
| "clip_ratio": 0.00027173911803402007, |
| "epoch": 0.06977061714909885, |
| "grad_norm": 0.0765228420495987, |
| "kl": 0.02642540223314427, |
| "learning_rate": 2.573684210526316e-06, |
| "loss": -0.0005, |
| "step": 511 |
| }, |
| { |
| "clip_ratio": 0.00027173911803402007, |
| "epoch": 0.06990715456034953, |
| "grad_norm": 0.07589668780565262, |
| "kl": 0.026321951998397708, |
| "learning_rate": 2.568421052631579e-06, |
| "loss": -0.0008, |
| "step": 512 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 125.25, |
| "epoch": 0.07004369197160022, |
| "grad_norm": 0.06559047847986221, |
| "kl": 0.026270317437592894, |
| "learning_rate": 2.5631578947368422e-06, |
| "loss": 0.0003, |
| "num_tokens": 1421669.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 513 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0701802293828509, |
| "grad_norm": 0.06494513899087906, |
| "kl": 0.026335061265854165, |
| "learning_rate": 2.5578947368421054e-06, |
| "loss": 0.0002, |
| "step": 514 |
| }, |
| { |
| "clip_ratio": 0.00020695364219136536, |
| "epoch": 0.07031676679410158, |
| "grad_norm": 0.06433810293674469, |
| "kl": 0.027359777392121032, |
| "learning_rate": 2.552631578947369e-06, |
| "loss": -0.0001, |
| "step": 515 |
| }, |
| { |
| "clip_ratio": 0.00020695364219136536, |
| "epoch": 0.07045330420535227, |
| "grad_norm": 0.06409961730241776, |
| "kl": 0.026299011806258932, |
| "learning_rate": 2.5473684210526317e-06, |
| "loss": -0.0002, |
| "step": 516 |
| }, |
| { |
| "clip_ratio": 0.0007239816040964797, |
| "completion_length": 146.0625, |
| "epoch": 0.07058984161660295, |
| "grad_norm": 0.10985471308231354, |
| "kl": 0.028434559819288552, |
| "learning_rate": 2.5421052631578953e-06, |
| "loss": 0.0, |
| "num_tokens": 1433875.0, |
| "reward": 0.578125, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.578125, |
| "step": 517 |
| }, |
| { |
| "clip_ratio": 0.0003907657810486853, |
| "epoch": 0.07072637902785363, |
| "grad_norm": 0.08845436573028564, |
| "kl": 0.029101567299221642, |
| "learning_rate": 2.536842105263158e-06, |
| "loss": -0.0001, |
| "step": 518 |
| }, |
| { |
| "clip_ratio": 0.0006556590233230963, |
| "epoch": 0.07086291643910432, |
| "grad_norm": 0.0867236852645874, |
| "kl": 0.028989310667384416, |
| "learning_rate": 2.5315789473684212e-06, |
| "loss": -0.0004, |
| "step": 519 |
| }, |
| { |
| "clip_ratio": 0.0002281021879753098, |
| "epoch": 0.070999453850355, |
| "grad_norm": 0.08522820472717285, |
| "kl": 0.029066309274639934, |
| "learning_rate": 2.5263157894736844e-06, |
| "loss": -0.001, |
| "step": 520 |
| }, |
| { |
| "clip_ratio": 0.0005073317588539794, |
| "completion_length": 137.53125, |
| "epoch": 0.07113599126160568, |
| "grad_norm": 0.08292447030544281, |
| "kl": 0.02251457070815377, |
| "learning_rate": 2.5210526315789475e-06, |
| "loss": 0.0005, |
| "num_tokens": 1444744.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 521 |
| }, |
| { |
| "clip_ratio": 0.00021258502965793014, |
| "epoch": 0.07127252867285637, |
| "grad_norm": 0.0836159959435463, |
| "kl": 0.022670210397336632, |
| "learning_rate": 2.5157894736842107e-06, |
| "loss": -0.0001, |
| "step": 522 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07140906608410705, |
| "grad_norm": 0.08502236008644104, |
| "kl": 0.023073405434843153, |
| "learning_rate": 2.510526315789474e-06, |
| "loss": -0.0004, |
| "step": 523 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07154560349535773, |
| "grad_norm": 0.08420410007238388, |
| "kl": 0.02333970883046277, |
| "learning_rate": 2.5052631578947375e-06, |
| "loss": -0.0008, |
| "step": 524 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 142.375, |
| "epoch": 0.0716821409066084, |
| "grad_norm": 0.06113371253013611, |
| "kl": 0.028267830901313573, |
| "learning_rate": 2.5e-06, |
| "loss": 0.0002, |
| "num_tokens": 1456204.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 525 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0718186783178591, |
| "grad_norm": 0.058986373245716095, |
| "kl": 0.029190031375037506, |
| "learning_rate": 2.4947368421052634e-06, |
| "loss": 0.0001, |
| "step": 526 |
| }, |
| { |
| "clip_ratio": 0.00019531250291038305, |
| "epoch": 0.07195521572910978, |
| "grad_norm": 0.056226328015327454, |
| "kl": 0.029348920157644898, |
| "learning_rate": 2.4894736842105265e-06, |
| "loss": -0.0001, |
| "step": 527 |
| }, |
| { |
| "clip_ratio": 0.0003004807804245502, |
| "epoch": 0.07209175314036045, |
| "grad_norm": 0.06096241623163223, |
| "kl": 0.029138524609152228, |
| "learning_rate": 2.4842105263157897e-06, |
| "loss": -0.0002, |
| "step": 528 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 140.0, |
| "epoch": 0.07222829055161115, |
| "grad_norm": 0.0643433928489685, |
| "kl": 0.024482485474436544, |
| "learning_rate": 2.478947368421053e-06, |
| "loss": 0.0002, |
| "num_tokens": 1468144.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 529 |
| }, |
| { |
| "clip_ratio": 0.00013185653369873762, |
| "epoch": 0.07236482796286182, |
| "grad_norm": 0.06284752488136292, |
| "kl": 0.024148478420102037, |
| "learning_rate": 2.473684210526316e-06, |
| "loss": 0.0001, |
| "step": 530 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0725013653741125, |
| "grad_norm": 0.055365268141031265, |
| "kl": 0.02333252050448209, |
| "learning_rate": 2.468421052631579e-06, |
| "loss": 0.0, |
| "step": 531 |
| }, |
| { |
| "clip_ratio": 0.0004308253701310605, |
| "epoch": 0.0726379027853632, |
| "grad_norm": 0.046432189643383026, |
| "kl": 0.023968403591425158, |
| "learning_rate": 2.4631578947368424e-06, |
| "loss": -0.0004, |
| "step": 532 |
| }, |
| { |
| "clip_ratio": 0.00023854961909819394, |
| "completion_length": 113.0, |
| "epoch": 0.07277444019661387, |
| "grad_norm": 0.10641506314277649, |
| "kl": 0.028071990760508925, |
| "learning_rate": 2.4578947368421055e-06, |
| "loss": 0.0001, |
| "num_tokens": 1479376.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 533 |
| }, |
| { |
| "clip_ratio": 0.0004978458891855553, |
| "epoch": 0.07291097760786455, |
| "grad_norm": 0.1005944311618805, |
| "kl": 0.026655245223082602, |
| "learning_rate": 2.4526315789473687e-06, |
| "loss": -0.0002, |
| "step": 534 |
| }, |
| { |
| "clip_ratio": 0.00044969827285967767, |
| "epoch": 0.07304751501911524, |
| "grad_norm": 0.10238411277532578, |
| "kl": 0.02741476362280082, |
| "learning_rate": 2.447368421052632e-06, |
| "loss": -0.0003, |
| "step": 535 |
| }, |
| { |
| "clip_ratio": 0.0004633697681128979, |
| "epoch": 0.07318405243036592, |
| "grad_norm": 0.09010997414588928, |
| "kl": 0.027337706153048202, |
| "learning_rate": 2.442105263157895e-06, |
| "loss": -0.001, |
| "step": 536 |
| }, |
| { |
| "clip_ratio": 0.0007482993096346036, |
| "completion_length": 141.15625, |
| "epoch": 0.0733205898416166, |
| "grad_norm": 0.05969298258423805, |
| "kl": 0.02480026413104497, |
| "learning_rate": 2.436842105263158e-06, |
| "loss": 0.0002, |
| "num_tokens": 1491869.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 537 |
| }, |
| { |
| "clip_ratio": 0.0005413736435002647, |
| "epoch": 0.07345712725286728, |
| "grad_norm": 0.06196073070168495, |
| "kl": 0.024416016967734322, |
| "learning_rate": 2.4315789473684213e-06, |
| "loss": 0.0, |
| "step": 538 |
| }, |
| { |
| "clip_ratio": 0.0009388480830239132, |
| "epoch": 0.07359366466411797, |
| "grad_norm": 0.05955655127763748, |
| "kl": 0.024354956054594368, |
| "learning_rate": 2.4263157894736845e-06, |
| "loss": -0.0001, |
| "step": 539 |
| }, |
| { |
| "clip_ratio": 0.00010557432688074186, |
| "epoch": 0.07373020207536865, |
| "grad_norm": 0.05420181155204773, |
| "kl": 0.02480731243849732, |
| "learning_rate": 2.4210526315789477e-06, |
| "loss": -0.0004, |
| "step": 540 |
| }, |
| { |
| "clip_ratio": 0.0003166591050103307, |
| "completion_length": 149.0, |
| "epoch": 0.07386673948661933, |
| "grad_norm": 0.0756562277674675, |
| "kl": 0.024679992871824652, |
| "learning_rate": 2.415789473684211e-06, |
| "loss": 0.0, |
| "num_tokens": 1504213.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 541 |
| }, |
| { |
| "clip_ratio": 0.0001755617995513603, |
| "epoch": 0.07400327689787002, |
| "grad_norm": 0.07468439638614655, |
| "kl": 0.024849631765391678, |
| "learning_rate": 2.410526315789474e-06, |
| "loss": -0.0001, |
| "step": 542 |
| }, |
| { |
| "clip_ratio": 0.0004983036633348092, |
| "epoch": 0.0741398143091207, |
| "grad_norm": 0.07378986477851868, |
| "kl": 0.025112065923167393, |
| "learning_rate": 2.4052631578947367e-06, |
| "loss": -0.0003, |
| "step": 543 |
| }, |
| { |
| "clip_ratio": 0.0004942677624057978, |
| "epoch": 0.07427635172037138, |
| "grad_norm": 0.07501641660928726, |
| "kl": 0.025077814352698624, |
| "learning_rate": 2.4000000000000003e-06, |
| "loss": -0.0006, |
| "step": 544 |
| }, |
| { |
| "clip_ratio": 0.0005071907507954165, |
| "completion_length": 122.625, |
| "epoch": 0.07441288913162207, |
| "grad_norm": 0.07435806840658188, |
| "kl": 0.02418363041942939, |
| "learning_rate": 2.3947368421052635e-06, |
| "loss": 0.0002, |
| "num_tokens": 1515297.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 545 |
| }, |
| { |
| "clip_ratio": 0.0008165966864908114, |
| "epoch": 0.07454942654287275, |
| "grad_norm": 0.07489731162786484, |
| "kl": 0.025440962228458375, |
| "learning_rate": 2.3894736842105266e-06, |
| "loss": 0.0001, |
| "step": 546 |
| }, |
| { |
| "clip_ratio": 0.0005071907507954165, |
| "epoch": 0.07468596395412343, |
| "grad_norm": 0.06941622495651245, |
| "kl": 0.025200976844644174, |
| "learning_rate": 2.38421052631579e-06, |
| "loss": -0.0002, |
| "step": 547 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "epoch": 0.07482250136537412, |
| "grad_norm": 0.06859351694583893, |
| "kl": 0.025588049815269187, |
| "learning_rate": 2.378947368421053e-06, |
| "loss": -0.0006, |
| "step": 548 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 135.15625, |
| "epoch": 0.0749590387766248, |
| "grad_norm": 0.06504392623901367, |
| "kl": 0.025377440004376695, |
| "learning_rate": 2.373684210526316e-06, |
| "loss": 0.0002, |
| "num_tokens": 1526610.0, |
| "reward": 0.49375003576278687, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.49375003576278687, |
| "step": 549 |
| }, |
| { |
| "clip_ratio": 0.0004585849674185738, |
| "epoch": 0.07509557618787548, |
| "grad_norm": 0.06409519910812378, |
| "kl": 0.025559124507708475, |
| "learning_rate": 2.368421052631579e-06, |
| "loss": 0.0002, |
| "step": 550 |
| }, |
| { |
| "clip_ratio": 0.00020833333837799728, |
| "epoch": 0.07523211359912615, |
| "grad_norm": 0.06408659368753433, |
| "kl": 0.025702771148644388, |
| "learning_rate": 2.363157894736842e-06, |
| "loss": -0.0, |
| "step": 551 |
| }, |
| { |
| "clip_ratio": 0.0006567885575350374, |
| "epoch": 0.07536865101037685, |
| "grad_norm": 0.06450217217206955, |
| "kl": 0.025717437063576654, |
| "learning_rate": 2.357894736842105e-06, |
| "loss": -0.0004, |
| "step": 552 |
| }, |
| { |
| "clip_ratio": 0.000879274244653061, |
| "completion_length": 136.71875, |
| "epoch": 0.07550518842162753, |
| "grad_norm": 0.1316654086112976, |
| "kl": 0.029629670636495575, |
| "learning_rate": 2.352631578947369e-06, |
| "loss": 0.0003, |
| "num_tokens": 1538437.0, |
| "reward": 0.625, |
| "reward_std": 0.4086301922798157, |
| "rewards/reward_fn": 0.625, |
| "step": 553 |
| }, |
| { |
| "clip_ratio": 0.000472686137072742, |
| "epoch": 0.0756417258328782, |
| "grad_norm": 0.1313183605670929, |
| "kl": 0.029478842378011905, |
| "learning_rate": 2.347368421052632e-06, |
| "loss": -0.0003, |
| "step": 554 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0757782632441289, |
| "grad_norm": 0.12120455503463745, |
| "kl": 0.029763573867967352, |
| "learning_rate": 2.342105263157895e-06, |
| "loss": -0.0009, |
| "step": 555 |
| }, |
| { |
| "clip_ratio": 0.00015703517419751734, |
| "epoch": 0.07591480065537957, |
| "grad_norm": 0.1141739934682846, |
| "kl": 0.030053666734602302, |
| "learning_rate": 2.3368421052631583e-06, |
| "loss": -0.0016, |
| "step": 556 |
| }, |
| { |
| "clip_ratio": 0.0008116767785395496, |
| "completion_length": 157.625, |
| "epoch": 0.07605133806663025, |
| "grad_norm": 0.08487264066934586, |
| "kl": 0.023374435782898217, |
| "learning_rate": 2.331578947368421e-06, |
| "loss": 0.0003, |
| "num_tokens": 1551185.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 557 |
| }, |
| { |
| "clip_ratio": 0.0012086032365914434, |
| "epoch": 0.07618787547788095, |
| "grad_norm": 0.0859953910112381, |
| "kl": 0.02305353432893753, |
| "learning_rate": 2.326315789473684e-06, |
| "loss": -0.0001, |
| "step": 558 |
| }, |
| { |
| "clip_ratio": 0.0003681144007714465, |
| "epoch": 0.07632441288913162, |
| "grad_norm": 0.08386844396591187, |
| "kl": 0.023537778324680403, |
| "learning_rate": 2.3210526315789473e-06, |
| "loss": -0.0003, |
| "step": 559 |
| }, |
| { |
| "clip_ratio": 0.0010179669770877808, |
| "epoch": 0.0764609503003823, |
| "grad_norm": 0.0760241448879242, |
| "kl": 0.023646087502129376, |
| "learning_rate": 2.3157894736842105e-06, |
| "loss": -0.0008, |
| "step": 560 |
| }, |
| { |
| "clip_ratio": 0.0006391926144715399, |
| "completion_length": 139.71875, |
| "epoch": 0.07659748771163298, |
| "grad_norm": 0.04080972447991371, |
| "kl": 0.022678025299683213, |
| "learning_rate": 2.310526315789474e-06, |
| "loss": 0.0003, |
| "num_tokens": 1562616.0, |
| "reward": 0.9125000238418579, |
| "reward_std": 0.05863019451498985, |
| "rewards/reward_fn": 0.9125000238418579, |
| "step": 561 |
| }, |
| { |
| "clip_ratio": 0.0006391926144715399, |
| "epoch": 0.07673402512288367, |
| "grad_norm": 0.039302002638578415, |
| "kl": 0.022627048514550552, |
| "learning_rate": 2.3052631578947373e-06, |
| "loss": 0.0003, |
| "step": 562 |
| }, |
| { |
| "clip_ratio": 0.0005716463201679289, |
| "epoch": 0.07687056253413435, |
| "grad_norm": 0.03550795093178749, |
| "kl": 0.02283687226008624, |
| "learning_rate": 2.3000000000000004e-06, |
| "loss": 0.0002, |
| "step": 563 |
| }, |
| { |
| "clip_ratio": 0.0003576610761228949, |
| "epoch": 0.07700709994538503, |
| "grad_norm": 0.035457585006952286, |
| "kl": 0.02243449864909053, |
| "learning_rate": 2.294736842105263e-06, |
| "loss": 0.0002, |
| "step": 564 |
| }, |
| { |
| "clip_ratio": 0.00026483050896786153, |
| "completion_length": 136.28125, |
| "epoch": 0.07714363735663572, |
| "grad_norm": 0.07386517524719238, |
| "kl": 0.024913390225265175, |
| "learning_rate": 2.2894736842105263e-06, |
| "loss": 0.0003, |
| "num_tokens": 1573761.0, |
| "reward": 0.6906249523162842, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.6906249523162842, |
| "step": 565 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0772801747678864, |
| "grad_norm": 0.07367141544818878, |
| "kl": 0.024486958689521998, |
| "learning_rate": 2.2842105263157895e-06, |
| "loss": 0.0001, |
| "step": 566 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07741671217913708, |
| "grad_norm": 0.07088012993335724, |
| "kl": 0.024814803357003257, |
| "learning_rate": 2.2789473684210527e-06, |
| "loss": -0.0002, |
| "step": 567 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07755324959038777, |
| "grad_norm": 0.07038354128599167, |
| "kl": 0.02420603678910993, |
| "learning_rate": 2.273684210526316e-06, |
| "loss": -0.0004, |
| "step": 568 |
| }, |
| { |
| "clip_ratio": 0.0006139783363323659, |
| "completion_length": 137.46875, |
| "epoch": 0.07768978700163845, |
| "grad_norm": 0.09878756105899811, |
| "kl": 0.03216486514429562, |
| "learning_rate": 2.268421052631579e-06, |
| "loss": 0.0002, |
| "num_tokens": 1585816.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.2511056959629059, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 569 |
| }, |
| { |
| "clip_ratio": 0.0009651019354350865, |
| "epoch": 0.07782632441288913, |
| "grad_norm": 0.09815220534801483, |
| "kl": 0.032273774879286066, |
| "learning_rate": 2.2631578947368426e-06, |
| "loss": -0.0, |
| "step": 570 |
| }, |
| { |
| "clip_ratio": 0.0011762505891965702, |
| "epoch": 0.07796286182413982, |
| "grad_norm": 0.09831620007753372, |
| "kl": 0.03218546602874994, |
| "learning_rate": 2.2578947368421057e-06, |
| "loss": -0.0005, |
| "step": 571 |
| }, |
| { |
| "clip_ratio": 0.0012266866688150913, |
| "epoch": 0.0780993992353905, |
| "grad_norm": 0.0923735499382019, |
| "kl": 0.032335673458874226, |
| "learning_rate": 2.2526315789473685e-06, |
| "loss": -0.0007, |
| "step": 572 |
| }, |
| { |
| "clip_ratio": 0.0005346590478438884, |
| "completion_length": 128.40625, |
| "epoch": 0.07823593664664118, |
| "grad_norm": 0.07317294925451279, |
| "kl": 0.031242430064594373, |
| "learning_rate": 2.2473684210526316e-06, |
| "loss": 0.0003, |
| "num_tokens": 1597009.0, |
| "reward": 0.9156249761581421, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.9156249761581421, |
| "step": 573 |
| }, |
| { |
| "clip_ratio": 0.0001516990305390209, |
| "epoch": 0.07837247405789186, |
| "grad_norm": 0.0728185847401619, |
| "kl": 0.03100292361341417, |
| "learning_rate": 2.242105263157895e-06, |
| "loss": 0.0, |
| "step": 574 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07850901146914255, |
| "grad_norm": 0.07220152020454407, |
| "kl": 0.03159632673487067, |
| "learning_rate": 2.236842105263158e-06, |
| "loss": -0.0002, |
| "step": 575 |
| }, |
| { |
| "clip_ratio": 0.0003676470660138875, |
| "epoch": 0.07864554888039323, |
| "grad_norm": 0.0730409100651741, |
| "kl": 0.03174605511594564, |
| "learning_rate": 2.231578947368421e-06, |
| "loss": -0.0004, |
| "step": 576 |
| }, |
| { |
| "clip_ratio": 0.0003221649385523051, |
| "completion_length": 134.53125, |
| "epoch": 0.0787820862916439, |
| "grad_norm": 0.08895674347877502, |
| "kl": 0.027897415682673454, |
| "learning_rate": 2.2263157894736843e-06, |
| "loss": 0.0004, |
| "num_tokens": 1608962.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.578125, |
| "step": 577 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0789186237028946, |
| "grad_norm": 0.0873054563999176, |
| "kl": 0.027481836776132695, |
| "learning_rate": 2.221052631578948e-06, |
| "loss": 0.0001, |
| "step": 578 |
| }, |
| { |
| "clip_ratio": 0.0005704920913558453, |
| "epoch": 0.07905516111414528, |
| "grad_norm": 0.1051458939909935, |
| "kl": 0.0269019435509108, |
| "learning_rate": 2.2157894736842106e-06, |
| "loss": -0.0002, |
| "step": 579 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07919169852539595, |
| "grad_norm": 0.08494027704000473, |
| "kl": 0.026989290417986922, |
| "learning_rate": 2.2105263157894738e-06, |
| "loss": -0.0007, |
| "step": 580 |
| }, |
| { |
| "clip_ratio": 0.0010808736260514706, |
| "completion_length": 147.8125, |
| "epoch": 0.07932823593664665, |
| "grad_norm": 0.08235741406679153, |
| "kl": 0.025487731574685313, |
| "learning_rate": 2.205263157894737e-06, |
| "loss": 0.0005, |
| "num_tokens": 1621644.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.2337018847465515, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 581 |
| }, |
| { |
| "clip_ratio": 0.000671143236104399, |
| "epoch": 0.07946477334789732, |
| "grad_norm": 0.08296383172273636, |
| "kl": 0.02604247519047931, |
| "learning_rate": 2.2e-06, |
| "loss": 0.0002, |
| "step": 582 |
| }, |
| { |
| "clip_ratio": 0.00048515741946175694, |
| "epoch": 0.079601310759148, |
| "grad_norm": 0.08231493830680847, |
| "kl": 0.025760373420780525, |
| "learning_rate": 2.1947368421052633e-06, |
| "loss": -0.0001, |
| "step": 583 |
| }, |
| { |
| "clip_ratio": 0.00028742906579282135, |
| "epoch": 0.0797378481703987, |
| "grad_norm": 0.08307528495788574, |
| "kl": 0.026076595226186328, |
| "learning_rate": 2.1894736842105264e-06, |
| "loss": -0.0005, |
| "step": 584 |
| }, |
| { |
| "clip_ratio": 0.00041828946268651634, |
| "completion_length": 152.0625, |
| "epoch": 0.07987438558164937, |
| "grad_norm": 0.07609320431947708, |
| "kl": 0.027080167375970632, |
| "learning_rate": 2.1842105263157896e-06, |
| "loss": 0.0002, |
| "num_tokens": 1633538.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 585 |
| }, |
| { |
| "clip_ratio": 0.00020424836839083582, |
| "epoch": 0.08001092299290005, |
| "grad_norm": 0.07597316056489944, |
| "kl": 0.027335600112564862, |
| "learning_rate": 2.1789473684210528e-06, |
| "loss": 0.0001, |
| "step": 586 |
| }, |
| { |
| "clip_ratio": 0.00016711230273358524, |
| "epoch": 0.08014746040415073, |
| "grad_norm": 0.0741892158985138, |
| "kl": 0.027281122165732086, |
| "learning_rate": 2.173684210526316e-06, |
| "loss": -0.0001, |
| "step": 587 |
| }, |
| { |
| "clip_ratio": 0.000540279594133608, |
| "epoch": 0.08028399781540142, |
| "grad_norm": 0.0735078901052475, |
| "kl": 0.027697552199242637, |
| "learning_rate": 2.168421052631579e-06, |
| "loss": -0.0003, |
| "step": 588 |
| }, |
| { |
| "clip_ratio": 0.0004656559322029352, |
| "completion_length": 130.5, |
| "epoch": 0.0804205352266521, |
| "grad_norm": 0.0796666294336319, |
| "kl": 0.028533839125884697, |
| "learning_rate": 2.1631578947368423e-06, |
| "loss": 0.0003, |
| "num_tokens": 1645442.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 589 |
| }, |
| { |
| "clip_ratio": 0.00030637255986221135, |
| "epoch": 0.08055707263790278, |
| "grad_norm": 0.08065035194158554, |
| "kl": 0.02866685314802453, |
| "learning_rate": 2.1578947368421054e-06, |
| "loss": 0.0001, |
| "step": 590 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08069361004915347, |
| "grad_norm": 0.08036067336797714, |
| "kl": 0.029090011259540915, |
| "learning_rate": 2.1526315789473686e-06, |
| "loss": -0.0003, |
| "step": 591 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08083014746040415, |
| "grad_norm": 0.07513322681188583, |
| "kl": 0.028922376601258293, |
| "learning_rate": 2.1473684210526317e-06, |
| "loss": -0.0006, |
| "step": 592 |
| }, |
| { |
| "clip_ratio": 0.0009937883296515793, |
| "completion_length": 136.40625, |
| "epoch": 0.08096668487165483, |
| "grad_norm": 0.10887999832630157, |
| "kl": 0.025917158054653555, |
| "learning_rate": 2.142105263157895e-06, |
| "loss": 0.0004, |
| "num_tokens": 1657179.0, |
| "reward": 0.71875, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.71875, |
| "step": 593 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08110322228290552, |
| "grad_norm": 0.10297057032585144, |
| "kl": 0.025241338269552216, |
| "learning_rate": 2.136842105263158e-06, |
| "loss": 0.0001, |
| "step": 594 |
| }, |
| { |
| "clip_ratio": 0.0005030487664043903, |
| "epoch": 0.0812397596941562, |
| "grad_norm": 0.09531421959400177, |
| "kl": 0.026220592786557972, |
| "learning_rate": 2.1315789473684212e-06, |
| "loss": -0.0004, |
| "step": 595 |
| }, |
| { |
| "clip_ratio": 0.00044461380457505584, |
| "epoch": 0.08137629710540688, |
| "grad_norm": 0.08854269981384277, |
| "kl": 0.02591939302510582, |
| "learning_rate": 2.1263157894736844e-06, |
| "loss": -0.0008, |
| "step": 596 |
| }, |
| { |
| "clip_ratio": 0.00040286114381160587, |
| "completion_length": 138.15625, |
| "epoch": 0.08151283451665757, |
| "grad_norm": 0.08875960856676102, |
| "kl": 0.026444266317412257, |
| "learning_rate": 2.1210526315789476e-06, |
| "loss": -0.0002, |
| "num_tokens": 1669548.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 597 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08164937192790825, |
| "grad_norm": 0.08832712471485138, |
| "kl": 0.02660306473262608, |
| "learning_rate": 2.1157894736842107e-06, |
| "loss": -0.0003, |
| "step": 598 |
| }, |
| { |
| "clip_ratio": 0.00022163119865581393, |
| "epoch": 0.08178590933915893, |
| "grad_norm": 0.08737774938344955, |
| "kl": 0.02694275981048122, |
| "learning_rate": 2.110526315789474e-06, |
| "loss": -0.0007, |
| "step": 599 |
| }, |
| { |
| "clip_ratio": 0.00033734142198227346, |
| "epoch": 0.0819224467504096, |
| "grad_norm": 0.08558928221464157, |
| "kl": 0.02657399495365098, |
| "learning_rate": 2.105263157894737e-06, |
| "loss": -0.001, |
| "step": 600 |
| }, |
| { |
| "clip_ratio": 0.0001188212918350473, |
| "completion_length": 128.34375, |
| "epoch": 0.0820589841616603, |
| "grad_norm": 0.06710242480039597, |
| "kl": 0.030188038130290806, |
| "learning_rate": 2.1000000000000002e-06, |
| "loss": 0.0003, |
| "num_tokens": 1680927.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 601 |
| }, |
| { |
| "clip_ratio": 0.00032577493402641267, |
| "epoch": 0.08219552157291098, |
| "grad_norm": 0.06653324514627457, |
| "kl": 0.030682684533530846, |
| "learning_rate": 2.0947368421052634e-06, |
| "loss": 0.0003, |
| "step": 602 |
| }, |
| { |
| "clip_ratio": 0.00046111433766782284, |
| "epoch": 0.08233205898416165, |
| "grad_norm": 0.06612059473991394, |
| "kl": 0.030843880609609187, |
| "learning_rate": 2.0894736842105266e-06, |
| "loss": 0.0002, |
| "step": 603 |
| }, |
| { |
| "clip_ratio": 0.00020695364219136536, |
| "epoch": 0.08246859639541235, |
| "grad_norm": 0.06393194198608398, |
| "kl": 0.031093561818124726, |
| "learning_rate": 2.0842105263157897e-06, |
| "loss": -0.0001, |
| "step": 604 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "completion_length": 128.9375, |
| "epoch": 0.08260513380666303, |
| "grad_norm": 0.06816224753856659, |
| "kl": 0.026418528461363167, |
| "learning_rate": 2.078947368421053e-06, |
| "loss": 0.0003, |
| "num_tokens": 1691557.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 605 |
| }, |
| { |
| "clip_ratio": 0.0008448271255474538, |
| "epoch": 0.0827416712179137, |
| "grad_norm": 0.06632911413908005, |
| "kl": 0.0269781686947681, |
| "learning_rate": 2.073684210526316e-06, |
| "loss": 0.0003, |
| "step": 606 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.0828782086291644, |
| "grad_norm": 0.06394495069980621, |
| "kl": 0.026145196170546114, |
| "learning_rate": 2.068421052631579e-06, |
| "loss": -0.0001, |
| "step": 607 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.08301474604041507, |
| "grad_norm": 0.06764407455921173, |
| "kl": 0.026354748668381944, |
| "learning_rate": 2.0631578947368424e-06, |
| "loss": -0.0002, |
| "step": 608 |
| }, |
| { |
| "clip_ratio": 0.0009424716117791831, |
| "completion_length": 151.625, |
| "epoch": 0.08315128345166575, |
| "grad_norm": 0.08172643929719925, |
| "kl": 0.026212202908936888, |
| "learning_rate": 2.0578947368421055e-06, |
| "loss": 0.0005, |
| "num_tokens": 1703877.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 609 |
| }, |
| { |
| "clip_ratio": 0.0007314500107895583, |
| "epoch": 0.08328782086291645, |
| "grad_norm": 0.08089722692966461, |
| "kl": 0.0268472250609193, |
| "learning_rate": 2.0526315789473687e-06, |
| "loss": 0.0004, |
| "step": 610 |
| }, |
| { |
| "clip_ratio": 0.0004393939452711493, |
| "epoch": 0.08342435827416712, |
| "grad_norm": 0.08089084923267365, |
| "kl": 0.02699510075035505, |
| "learning_rate": 2.047368421052632e-06, |
| "loss": 0.0, |
| "step": 611 |
| }, |
| { |
| "clip_ratio": 0.0006021543667884544, |
| "epoch": 0.0835608956854178, |
| "grad_norm": 0.08019546419382095, |
| "kl": 0.026255997800035402, |
| "learning_rate": 2.042105263157895e-06, |
| "loss": -0.0005, |
| "step": 612 |
| }, |
| { |
| "clip_ratio": 0.000810613390058279, |
| "completion_length": 124.96875, |
| "epoch": 0.08369743309666848, |
| "grad_norm": 0.09329909086227417, |
| "kl": 0.026048675848869607, |
| "learning_rate": 2.036842105263158e-06, |
| "loss": 0.0001, |
| "num_tokens": 1714476.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.25980761647224426, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 613 |
| }, |
| { |
| "clip_ratio": 0.0012707302666967735, |
| "epoch": 0.08383397050791917, |
| "grad_norm": 0.09495878219604492, |
| "kl": 0.025636921141995117, |
| "learning_rate": 2.031578947368421e-06, |
| "loss": 0.0002, |
| "step": 614 |
| }, |
| { |
| "clip_ratio": 0.0001707650226308033, |
| "epoch": 0.08397050791916985, |
| "grad_norm": 0.09412883967161179, |
| "kl": 0.025533305306453258, |
| "learning_rate": 2.026315789473684e-06, |
| "loss": -0.0005, |
| "step": 615 |
| }, |
| { |
| "clip_ratio": 0.00100363623641897, |
| "epoch": 0.08410704533042053, |
| "grad_norm": 0.08954595029354095, |
| "kl": 0.02580202795797959, |
| "learning_rate": 2.0210526315789477e-06, |
| "loss": -0.0008, |
| "step": 616 |
| }, |
| { |
| "clip_ratio": 0.0007753242534818128, |
| "completion_length": 168.53125, |
| "epoch": 0.08424358274167122, |
| "grad_norm": 0.07307417690753937, |
| "kl": 0.026822772080777213, |
| "learning_rate": 2.015789473684211e-06, |
| "loss": 0.0002, |
| "num_tokens": 1727825.0, |
| "reward": 0.578125, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.578125, |
| "step": 617 |
| }, |
| { |
| "clip_ratio": 0.0007655591471120715, |
| "epoch": 0.0843801201529219, |
| "grad_norm": 0.0729583203792572, |
| "kl": 0.02734056010376662, |
| "learning_rate": 2.010526315789474e-06, |
| "loss": 0.0002, |
| "step": 618 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08451665756417258, |
| "grad_norm": 0.07249397039413452, |
| "kl": 0.027082344196969643, |
| "learning_rate": 2.005263157894737e-06, |
| "loss": -0.0001, |
| "step": 619 |
| }, |
| { |
| "clip_ratio": 0.0004055017634527758, |
| "epoch": 0.08465319497542327, |
| "grad_norm": 0.06983788311481476, |
| "kl": 0.027317771135130897, |
| "learning_rate": 2.0000000000000003e-06, |
| "loss": -0.0003, |
| "step": 620 |
| }, |
| { |
| "clip_ratio": 0.00041753742698347196, |
| "completion_length": 153.78125, |
| "epoch": 0.08478973238667395, |
| "grad_norm": 0.0808953270316124, |
| "kl": 0.028358329698676243, |
| "learning_rate": 1.994736842105263e-06, |
| "loss": 0.0004, |
| "num_tokens": 1740298.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 621 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "epoch": 0.08492626979792463, |
| "grad_norm": 0.08027170598506927, |
| "kl": 0.02871102059725672, |
| "learning_rate": 1.9894736842105262e-06, |
| "loss": 0.0, |
| "step": 622 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "epoch": 0.08506280720917532, |
| "grad_norm": 0.07950595021247864, |
| "kl": 0.02856921256170608, |
| "learning_rate": 1.9842105263157894e-06, |
| "loss": -0.0001, |
| "step": 623 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "epoch": 0.085199344620426, |
| "grad_norm": 0.07559674233198166, |
| "kl": 0.02868326409952715, |
| "learning_rate": 1.978947368421053e-06, |
| "loss": -0.0006, |
| "step": 624 |
| }, |
| { |
| "clip_ratio": 0.00043604651000350714, |
| "completion_length": 137.78125, |
| "epoch": 0.08533588203167668, |
| "grad_norm": 0.03387494385242462, |
| "kl": 0.025424274819670245, |
| "learning_rate": 1.973684210526316e-06, |
| "loss": 0.0002, |
| "num_tokens": 1752347.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 625 |
| }, |
| { |
| "clip_ratio": 0.0005163294408703223, |
| "epoch": 0.08547241944292736, |
| "grad_norm": 0.03339214622974396, |
| "kl": 0.025453351787291467, |
| "learning_rate": 1.9684210526315793e-06, |
| "loss": 0.0002, |
| "step": 626 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08560895685417805, |
| "grad_norm": 0.033701345324516296, |
| "kl": 0.025396857847226784, |
| "learning_rate": 1.9631578947368425e-06, |
| "loss": 0.0002, |
| "step": 627 |
| }, |
| { |
| "clip_ratio": 0.0001453488366678357, |
| "epoch": 0.08574549426542873, |
| "grad_norm": 0.03361152485013008, |
| "kl": 0.02518440838321112, |
| "learning_rate": 1.9578947368421052e-06, |
| "loss": 0.0, |
| "step": 628 |
| }, |
| { |
| "clip_ratio": 0.0002741228090599179, |
| "completion_length": 124.375, |
| "epoch": 0.0858820316766794, |
| "grad_norm": 0.09590297192335129, |
| "kl": 0.03190047410316765, |
| "learning_rate": 1.9526315789473684e-06, |
| "loss": -0.0001, |
| "num_tokens": 1763875.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 629 |
| }, |
| { |
| "clip_ratio": 0.0007283711893251166, |
| "epoch": 0.0860185690879301, |
| "grad_norm": 0.09830604493618011, |
| "kl": 0.03203440588549711, |
| "learning_rate": 1.9473684210526315e-06, |
| "loss": -0.0, |
| "step": 630 |
| }, |
| { |
| "clip_ratio": 0.0004833552229683846, |
| "epoch": 0.08615510649918078, |
| "grad_norm": 0.09596423804759979, |
| "kl": 0.03181770283845253, |
| "learning_rate": 1.9421052631578947e-06, |
| "loss": -0.0004, |
| "step": 631 |
| }, |
| { |
| "clip_ratio": 0.0006497451540781185, |
| "epoch": 0.08629164391043145, |
| "grad_norm": 0.08988839387893677, |
| "kl": 0.031553027947666124, |
| "learning_rate": 1.936842105263158e-06, |
| "loss": -0.0008, |
| "step": 632 |
| }, |
| { |
| "clip_ratio": 0.0003396739193703979, |
| "completion_length": 139.6875, |
| "epoch": 0.08642818132168215, |
| "grad_norm": 0.11786044389009476, |
| "kl": 0.029205120386905037, |
| "learning_rate": 1.9315789473684215e-06, |
| "loss": 0.0001, |
| "num_tokens": 1775645.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 633 |
| }, |
| { |
| "clip_ratio": 0.0007215468504000455, |
| "epoch": 0.08656471873293282, |
| "grad_norm": 0.117555171251297, |
| "kl": 0.029039909320999868, |
| "learning_rate": 1.9263157894736846e-06, |
| "loss": 0.0, |
| "step": 634 |
| }, |
| { |
| "clip_ratio": 0.00045773174497298896, |
| "epoch": 0.0867012561441835, |
| "grad_norm": 0.11143205314874649, |
| "kl": 0.029505190832423978, |
| "learning_rate": 1.9210526315789474e-06, |
| "loss": -0.0005, |
| "step": 635 |
| }, |
| { |
| "clip_ratio": 0.0006555165600730106, |
| "epoch": 0.0868377935554342, |
| "grad_norm": 0.10721877217292786, |
| "kl": 0.02906324692594353, |
| "learning_rate": 1.9157894736842105e-06, |
| "loss": -0.0012, |
| "step": 636 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 127.125, |
| "epoch": 0.08697433096668487, |
| "grad_norm": 0.0816088318824768, |
| "kl": 0.03337540605571121, |
| "learning_rate": 1.9105263157894737e-06, |
| "loss": 0.0002, |
| "num_tokens": 1787193.0, |
| "reward": 0.578125, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.578125, |
| "step": 637 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08711086837793555, |
| "grad_norm": 0.08079145103693008, |
| "kl": 0.03298256819834933, |
| "learning_rate": 1.905263157894737e-06, |
| "loss": 0.0002, |
| "step": 638 |
| }, |
| { |
| "clip_ratio": 0.00028935185400769114, |
| "epoch": 0.08724740578918623, |
| "grad_norm": 0.07375474274158478, |
| "kl": 0.03400870703626424, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": -0.0002, |
| "step": 639 |
| }, |
| { |
| "clip_ratio": 0.00014952152559999377, |
| "epoch": 0.08738394320043692, |
| "grad_norm": 0.07169979810714722, |
| "kl": 0.0339630561938975, |
| "learning_rate": 1.8947368421052634e-06, |
| "loss": -0.0003, |
| "step": 640 |
| }, |
| { |
| "clip_ratio": 0.00046546234807465225, |
| "completion_length": 129.09375, |
| "epoch": 0.0875204806116876, |
| "grad_norm": 0.0885995551943779, |
| "kl": 0.03377768350765109, |
| "learning_rate": 1.8894736842105266e-06, |
| "loss": 0.0002, |
| "num_tokens": 1798452.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 641 |
| }, |
| { |
| "clip_ratio": 0.0004482519725570455, |
| "epoch": 0.08765701802293828, |
| "grad_norm": 0.0879860669374466, |
| "kl": 0.0345135809329804, |
| "learning_rate": 1.8842105263157895e-06, |
| "loss": 0.0, |
| "step": 642 |
| }, |
| { |
| "clip_ratio": 0.00014810427092015743, |
| "epoch": 0.08779355543418897, |
| "grad_norm": 0.08827782422304153, |
| "kl": 0.03430288049275987, |
| "learning_rate": 1.8789473684210527e-06, |
| "loss": -0.0001, |
| "step": 643 |
| }, |
| { |
| "clip_ratio": 0.0009655518224462867, |
| "epoch": 0.08793009284543965, |
| "grad_norm": 0.0869118869304657, |
| "kl": 0.03385839582188055, |
| "learning_rate": 1.8736842105263158e-06, |
| "loss": -0.0003, |
| "step": 644 |
| }, |
| { |
| "clip_ratio": 0.001409449425409548, |
| "completion_length": 128.125, |
| "epoch": 0.08806663025669033, |
| "grad_norm": 0.0921865850687027, |
| "kl": 0.033534372836584225, |
| "learning_rate": 1.868421052631579e-06, |
| "loss": 0.0005, |
| "num_tokens": 1809316.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 645 |
| }, |
| { |
| "clip_ratio": 0.0006227420526556671, |
| "epoch": 0.08820316766794102, |
| "grad_norm": 0.09347839653491974, |
| "kl": 0.03369360527722165, |
| "learning_rate": 1.8631578947368424e-06, |
| "loss": 0.0, |
| "step": 646 |
| }, |
| { |
| "clip_ratio": 0.0008524778095306829, |
| "epoch": 0.0883397050791917, |
| "grad_norm": 0.08786574006080627, |
| "kl": 0.03364496599533595, |
| "learning_rate": 1.8578947368421055e-06, |
| "loss": -0.0002, |
| "step": 647 |
| }, |
| { |
| "clip_ratio": 0.0004562043759506196, |
| "epoch": 0.08847624249044238, |
| "grad_norm": 0.08518879115581512, |
| "kl": 0.03309225430712104, |
| "learning_rate": 1.8526315789473687e-06, |
| "loss": -0.0008, |
| "step": 648 |
| }, |
| { |
| "clip_ratio": 0.0007344547484535724, |
| "completion_length": 130.65625, |
| "epoch": 0.08861277990169307, |
| "grad_norm": 0.093144990503788, |
| "kl": 0.037557946430752054, |
| "learning_rate": 1.8473684210526319e-06, |
| "loss": 0.0008, |
| "num_tokens": 1820961.0, |
| "reward": 0.6906249523162842, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.6906249523162842, |
| "step": 649 |
| }, |
| { |
| "clip_ratio": 0.0004762095195474103, |
| "epoch": 0.08874931731294375, |
| "grad_norm": 0.09195293486118317, |
| "kl": 0.03590751811861992, |
| "learning_rate": 1.8421052631578948e-06, |
| "loss": 0.0003, |
| "step": 650 |
| }, |
| { |
| "clip_ratio": 0.000642394064925611, |
| "epoch": 0.08888585472419443, |
| "grad_norm": 0.09232187271118164, |
| "kl": 0.03571216404088773, |
| "learning_rate": 1.836842105263158e-06, |
| "loss": 0.0003, |
| "step": 651 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0890223921354451, |
| "grad_norm": 0.08797423541545868, |
| "kl": 0.03472030599368736, |
| "learning_rate": 1.8315789473684211e-06, |
| "loss": -0.0004, |
| "step": 652 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 149.21875, |
| "epoch": 0.0891589295466958, |
| "grad_norm": 0.04103375971317291, |
| "kl": 0.03244091695523821, |
| "learning_rate": 1.8263157894736843e-06, |
| "loss": 0.0003, |
| "num_tokens": 1833080.0, |
| "reward": 0.71875, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.71875, |
| "step": 653 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08929546695794648, |
| "grad_norm": 0.04076002910733223, |
| "kl": 0.03265755213215016, |
| "learning_rate": 1.8210526315789475e-06, |
| "loss": 0.0003, |
| "step": 654 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08943200436919715, |
| "grad_norm": 0.04100827872753143, |
| "kl": 0.03254827967612073, |
| "learning_rate": 1.8157894736842109e-06, |
| "loss": 0.0002, |
| "step": 655 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08956854178044785, |
| "grad_norm": 0.04075401648879051, |
| "kl": 0.03307641306309961, |
| "learning_rate": 1.810526315789474e-06, |
| "loss": 0.0, |
| "step": 656 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 147.78125, |
| "epoch": 0.08970507919169853, |
| "grad_norm": 0.03296128287911415, |
| "kl": 0.027622340683592483, |
| "learning_rate": 1.805263157894737e-06, |
| "loss": 0.0001, |
| "num_tokens": 1845289.0, |
| "reward": 0.859375, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.859375, |
| "step": 657 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0898416166029492, |
| "grad_norm": 0.03176380693912506, |
| "kl": 0.027481299359351397, |
| "learning_rate": 1.8000000000000001e-06, |
| "loss": 0.0001, |
| "step": 658 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0899781540141999, |
| "grad_norm": 0.031271521002054214, |
| "kl": 0.02771525498246774, |
| "learning_rate": 1.7947368421052633e-06, |
| "loss": 0.0001, |
| "step": 659 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09011469142545057, |
| "grad_norm": 0.03006802499294281, |
| "kl": 0.027794133289717138, |
| "learning_rate": 1.7894736842105265e-06, |
| "loss": 0.0, |
| "step": 660 |
| }, |
| { |
| "clip_ratio": 0.00037045805947855115, |
| "completion_length": 144.34375, |
| "epoch": 0.09025122883670125, |
| "grad_norm": 0.07770948857069016, |
| "kl": 0.032641842582961544, |
| "learning_rate": 1.7842105263157896e-06, |
| "loss": 0.0002, |
| "num_tokens": 1857092.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 661 |
| }, |
| { |
| "clip_ratio": 0.00035511364694684744, |
| "epoch": 0.09038776624795195, |
| "grad_norm": 0.08898326009511948, |
| "kl": 0.03239144055987708, |
| "learning_rate": 1.7789473684210528e-06, |
| "loss": 0.0003, |
| "step": 662 |
| }, |
| { |
| "clip_ratio": 0.0004056590114487335, |
| "epoch": 0.09052430365920262, |
| "grad_norm": 0.07720089703798294, |
| "kl": 0.03239328300696798, |
| "learning_rate": 1.7736842105263162e-06, |
| "loss": -0.0001, |
| "step": 663 |
| }, |
| { |
| "clip_ratio": 0.0003613803564803675, |
| "epoch": 0.0906608410704533, |
| "grad_norm": 0.0780443400144577, |
| "kl": 0.03282105067046359, |
| "learning_rate": 1.768421052631579e-06, |
| "loss": -0.0002, |
| "step": 664 |
| }, |
| { |
| "clip_ratio": 0.00037165620597079396, |
| "completion_length": 142.3125, |
| "epoch": 0.09079737848170398, |
| "grad_norm": 0.0875195860862732, |
| "kl": 0.032956337759969756, |
| "learning_rate": 1.7631578947368423e-06, |
| "loss": 0.0003, |
| "num_tokens": 1868814.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 665 |
| }, |
| { |
| "clip_ratio": 0.0006258496141526848, |
| "epoch": 0.09093391589295467, |
| "grad_norm": 0.08770208060741425, |
| "kl": 0.032618753350107, |
| "learning_rate": 1.7578947368421054e-06, |
| "loss": -0.0001, |
| "step": 666 |
| }, |
| { |
| "clip_ratio": 0.00048392938333563507, |
| "epoch": 0.09107045330420535, |
| "grad_norm": 0.086226686835289, |
| "kl": 0.03322019218467176, |
| "learning_rate": 1.7526315789473686e-06, |
| "loss": -0.0, |
| "step": 667 |
| }, |
| { |
| "clip_ratio": 0.0005311927088769153, |
| "epoch": 0.09120699071545603, |
| "grad_norm": 0.08620792627334595, |
| "kl": 0.032796544779557735, |
| "learning_rate": 1.7473684210526318e-06, |
| "loss": -0.0005, |
| "step": 668 |
| }, |
| { |
| "clip_ratio": 0.00016891892300918698, |
| "completion_length": 152.03125, |
| "epoch": 0.09134352812670672, |
| "grad_norm": 0.06065315380692482, |
| "kl": 0.030359022639459, |
| "learning_rate": 1.742105263157895e-06, |
| "loss": 0.0004, |
| "num_tokens": 1881131.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 669 |
| }, |
| { |
| "clip_ratio": 0.0001806358341127634, |
| "epoch": 0.0914800655379574, |
| "grad_norm": 0.060918476432561874, |
| "kl": 0.029836400091880932, |
| "learning_rate": 1.736842105263158e-06, |
| "loss": 0.0003, |
| "step": 670 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09161660294920808, |
| "grad_norm": 0.06018917262554169, |
| "kl": 0.029947946954052895, |
| "learning_rate": 1.731578947368421e-06, |
| "loss": 0.0, |
| "step": 671 |
| }, |
| { |
| "clip_ratio": 0.0001806358341127634, |
| "epoch": 0.09175314036045877, |
| "grad_norm": 0.06113429367542267, |
| "kl": 0.02997833924018778, |
| "learning_rate": 1.7263157894736842e-06, |
| "loss": -0.0001, |
| "step": 672 |
| }, |
| { |
| "clip_ratio": 0.0010184195562032983, |
| "completion_length": 149.71875, |
| "epoch": 0.09188967777170945, |
| "grad_norm": 0.11531484127044678, |
| "kl": 0.029697615158511326, |
| "learning_rate": 1.7210526315789474e-06, |
| "loss": 0.0002, |
| "num_tokens": 1893158.0, |
| "reward": 0.6031249761581421, |
| "reward_std": 0.42610567808151245, |
| "rewards/reward_fn": 0.6031249761581421, |
| "step": 673 |
| }, |
| { |
| "clip_ratio": 0.0008683160558575764, |
| "epoch": 0.09202621518296013, |
| "grad_norm": 0.11043091863393784, |
| "kl": 0.031451030779862776, |
| "learning_rate": 1.7157894736842107e-06, |
| "loss": 0.0, |
| "step": 674 |
| }, |
| { |
| "clip_ratio": 0.0016366316704079509, |
| "epoch": 0.09216275259421082, |
| "grad_norm": 0.11153166741132736, |
| "kl": 0.030584218620788306, |
| "learning_rate": 1.710526315789474e-06, |
| "loss": -0.0003, |
| "step": 675 |
| }, |
| { |
| "clip_ratio": 0.0007215101504698396, |
| "epoch": 0.0922992900054615, |
| "grad_norm": 0.10511306673288345, |
| "kl": 0.030299391684820876, |
| "learning_rate": 1.705263157894737e-06, |
| "loss": -0.0009, |
| "step": 676 |
| }, |
| { |
| "clip_ratio": 0.00036520417779684067, |
| "completion_length": 150.875, |
| "epoch": 0.09243582741671218, |
| "grad_norm": 0.10060536116361618, |
| "kl": 0.03141664047143422, |
| "learning_rate": 1.7000000000000002e-06, |
| "loss": 0.0006, |
| "num_tokens": 1905082.0, |
| "reward": 0.71875, |
| "reward_std": 0.2899519205093384, |
| "rewards/reward_fn": 0.71875, |
| "step": 677 |
| }, |
| { |
| "clip_ratio": 0.0008521379495505244, |
| "epoch": 0.09257236482796286, |
| "grad_norm": 0.0980759933590889, |
| "kl": 0.03153267546440475, |
| "learning_rate": 1.6947368421052632e-06, |
| "loss": 0.0001, |
| "step": 678 |
| }, |
| { |
| "clip_ratio": 0.0003382211725693196, |
| "epoch": 0.09270890223921355, |
| "grad_norm": 0.09468948096036911, |
| "kl": 0.03160182334249839, |
| "learning_rate": 1.6894736842105264e-06, |
| "loss": -0.0004, |
| "step": 679 |
| }, |
| { |
| "clip_ratio": 0.0003924800839740783, |
| "epoch": 0.09284543965046423, |
| "grad_norm": 0.09634365886449814, |
| "kl": 0.03162823195452802, |
| "learning_rate": 1.6842105263157895e-06, |
| "loss": -0.0007, |
| "step": 680 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.28125, |
| "epoch": 0.0929819770617149, |
| "grad_norm": 0.08576124906539917, |
| "kl": 0.029789241671096534, |
| "learning_rate": 1.6789473684210527e-06, |
| "loss": 0.0004, |
| "num_tokens": 1915887.0, |
| "reward": 0.9156249761581421, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.9156249761581421, |
| "step": 681 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0931185144729656, |
| "grad_norm": 0.08568418025970459, |
| "kl": 0.02953265013638884, |
| "learning_rate": 1.673684210526316e-06, |
| "loss": 0.0003, |
| "step": 682 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09325505188421628, |
| "grad_norm": 0.08653020858764648, |
| "kl": 0.030135785636957735, |
| "learning_rate": 1.6684210526315792e-06, |
| "loss": 0.0001, |
| "step": 683 |
| }, |
| { |
| "clip_ratio": 0.0004459502233657986, |
| "epoch": 0.09339158929546695, |
| "grad_norm": 0.07059013098478317, |
| "kl": 0.03075120344874449, |
| "learning_rate": 1.6631578947368424e-06, |
| "loss": -0.0001, |
| "step": 684 |
| }, |
| { |
| "clip_ratio": 0.0005163708701729774, |
| "completion_length": 137.875, |
| "epoch": 0.09352812670671765, |
| "grad_norm": 0.0986505001783371, |
| "kl": 0.03283812435984146, |
| "learning_rate": 1.6578947368421053e-06, |
| "loss": 0.0005, |
| "num_tokens": 1926891.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.578125, |
| "step": 685 |
| }, |
| { |
| "clip_ratio": 0.0004849253600696102, |
| "epoch": 0.09366466411796832, |
| "grad_norm": 0.09848025441169739, |
| "kl": 0.032939996555796824, |
| "learning_rate": 1.6526315789473685e-06, |
| "loss": 0.0002, |
| "step": 686 |
| }, |
| { |
| "clip_ratio": 0.0006144791841506958, |
| "epoch": 0.093801201529219, |
| "grad_norm": 0.09903288632631302, |
| "kl": 0.033225755047169514, |
| "learning_rate": 1.6473684210526317e-06, |
| "loss": 0.0, |
| "step": 687 |
| }, |
| { |
| "clip_ratio": 0.0003739090752787888, |
| "epoch": 0.0939377389404697, |
| "grad_norm": 0.09462776780128479, |
| "kl": 0.0339893525961088, |
| "learning_rate": 1.6421052631578948e-06, |
| "loss": -0.0003, |
| "step": 688 |
| }, |
| { |
| "clip_ratio": 0.0004387063963804394, |
| "completion_length": 140.9375, |
| "epoch": 0.09407427635172037, |
| "grad_norm": 0.07126783579587936, |
| "kl": 0.032499870838364586, |
| "learning_rate": 1.636842105263158e-06, |
| "loss": 0.0004, |
| "num_tokens": 1938977.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 689 |
| }, |
| { |
| "clip_ratio": 0.00033783784601837397, |
| "epoch": 0.09421081376297105, |
| "grad_norm": 0.0714046061038971, |
| "kl": 0.03266788163455203, |
| "learning_rate": 1.6315789473684212e-06, |
| "loss": 0.0001, |
| "step": 690 |
| }, |
| { |
| "clip_ratio": 0.0003198851045453921, |
| "epoch": 0.09434735117422173, |
| "grad_norm": 0.07156651467084885, |
| "kl": 0.03251979897322599, |
| "learning_rate": 1.6263157894736845e-06, |
| "loss": 0.0002, |
| "step": 691 |
| }, |
| { |
| "clip_ratio": 0.0002877402148442343, |
| "epoch": 0.09448388858547242, |
| "grad_norm": 0.07169118523597717, |
| "kl": 0.03235316323116422, |
| "learning_rate": 1.6210526315789473e-06, |
| "loss": -0.0001, |
| "step": 692 |
| }, |
| { |
| "clip_ratio": 0.00025826445198617876, |
| "completion_length": 133.65625, |
| "epoch": 0.0946204259967231, |
| "grad_norm": 0.0648774579167366, |
| "kl": 0.03120983185363002, |
| "learning_rate": 1.6157894736842106e-06, |
| "loss": 0.0002, |
| "num_tokens": 1950866.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 693 |
| }, |
| { |
| "clip_ratio": 0.00025826445198617876, |
| "epoch": 0.09475696340797378, |
| "grad_norm": 0.06372489035129547, |
| "kl": 0.0311219054274261, |
| "learning_rate": 1.6105263157894738e-06, |
| "loss": 0.0001, |
| "step": 694 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09489350081922447, |
| "grad_norm": 0.06404761970043182, |
| "kl": 0.030508951924275607, |
| "learning_rate": 1.605263157894737e-06, |
| "loss": -0.0, |
| "step": 695 |
| }, |
| { |
| "clip_ratio": 0.0002281021879753098, |
| "epoch": 0.09503003823047515, |
| "grad_norm": 0.0647362545132637, |
| "kl": 0.031124803936108947, |
| "learning_rate": 1.6000000000000001e-06, |
| "loss": -0.0001, |
| "step": 696 |
| }, |
| { |
| "clip_ratio": 0.0002855776983778924, |
| "completion_length": 135.40625, |
| "epoch": 0.09516657564172583, |
| "grad_norm": 0.069391630589962, |
| "kl": 0.03132415152504109, |
| "learning_rate": 1.5947368421052633e-06, |
| "loss": 0.0004, |
| "num_tokens": 1962359.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 697 |
| }, |
| { |
| "clip_ratio": 0.0005032864864915609, |
| "epoch": 0.09530311305297652, |
| "grad_norm": 0.07143551856279373, |
| "kl": 0.03164268011460081, |
| "learning_rate": 1.5894736842105265e-06, |
| "loss": 0.0005, |
| "step": 698 |
| }, |
| { |
| "clip_ratio": 0.000163612567121163, |
| "epoch": 0.0954396504642272, |
| "grad_norm": 0.06680339574813843, |
| "kl": 0.03151996951783076, |
| "learning_rate": 1.5842105263157894e-06, |
| "loss": 0.0002, |
| "step": 699 |
| }, |
| { |
| "clip_ratio": 0.000163612567121163, |
| "epoch": 0.09557618787547788, |
| "grad_norm": 0.0664854645729065, |
| "kl": 0.03106894381926395, |
| "learning_rate": 1.5789473684210526e-06, |
| "loss": -0.0, |
| "step": 700 |
| }, |
| { |
| "clip_ratio": 0.0014787253021495417, |
| "completion_length": 166.3125, |
| "epoch": 0.09571272528672857, |
| "grad_norm": 0.11290665715932846, |
| "kl": 0.031161494320258498, |
| "learning_rate": 1.573684210526316e-06, |
| "loss": 0.0007, |
| "num_tokens": 1975669.0, |
| "reward": 0.578125, |
| "reward_std": 0.36360570788383484, |
| "rewards/reward_fn": 0.578125, |
| "step": 701 |
| }, |
| { |
| "clip_ratio": 0.000746544887078926, |
| "epoch": 0.09584926269797925, |
| "grad_norm": 0.11650830507278442, |
| "kl": 0.03087172904633917, |
| "learning_rate": 1.5684210526315791e-06, |
| "loss": 0.0003, |
| "step": 702 |
| }, |
| { |
| "clip_ratio": 0.0007508412381866947, |
| "epoch": 0.09598580010922993, |
| "grad_norm": 0.11172571778297424, |
| "kl": 0.03082931600511074, |
| "learning_rate": 1.5631578947368423e-06, |
| "loss": -0.0001, |
| "step": 703 |
| }, |
| { |
| "clip_ratio": 0.0008415376505581662, |
| "epoch": 0.0961223375204806, |
| "grad_norm": 0.11011453717947006, |
| "kl": 0.030562146741431206, |
| "learning_rate": 1.5578947368421054e-06, |
| "loss": -0.0005, |
| "step": 704 |
| }, |
| { |
| "clip_ratio": 0.000691575522068888, |
| "completion_length": 119.21875, |
| "epoch": 0.0962588749317313, |
| "grad_norm": 0.09140753000974655, |
| "kl": 0.03985722205834463, |
| "learning_rate": 1.5526315789473686e-06, |
| "loss": 0.0005, |
| "num_tokens": 1986412.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 705 |
| }, |
| { |
| "clip_ratio": 0.00048280542250722647, |
| "epoch": 0.09639541234298198, |
| "grad_norm": 0.09256947040557861, |
| "kl": 0.03949978741002269, |
| "learning_rate": 1.5473684210526316e-06, |
| "loss": 0.0003, |
| "step": 706 |
| }, |
| { |
| "clip_ratio": 0.0006655539618805051, |
| "epoch": 0.09653194975423265, |
| "grad_norm": 0.0919048860669136, |
| "kl": 0.04003440952510573, |
| "learning_rate": 1.5421052631578947e-06, |
| "loss": 0.0001, |
| "step": 707 |
| }, |
| { |
| "clip_ratio": 0.0002367424312978983, |
| "epoch": 0.09666848716548335, |
| "grad_norm": 0.08757032454013824, |
| "kl": 0.0402066423848737, |
| "learning_rate": 1.5368421052631579e-06, |
| "loss": -0.0003, |
| "step": 708 |
| }, |
| { |
| "clip_ratio": 0.00018601190822664648, |
| "completion_length": 131.5625, |
| "epoch": 0.09680502457673403, |
| "grad_norm": 0.04012110456824303, |
| "kl": 0.026871845373534597, |
| "learning_rate": 1.5315789473684213e-06, |
| "loss": 0.0003, |
| "num_tokens": 1997494.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 709 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0969415619879847, |
| "grad_norm": 0.04035944491624832, |
| "kl": 0.026953702850732952, |
| "learning_rate": 1.5263157894736844e-06, |
| "loss": 0.0002, |
| "step": 710 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0970780993992354, |
| "grad_norm": 0.040319595485925674, |
| "kl": 0.026927403203444555, |
| "learning_rate": 1.5210526315789476e-06, |
| "loss": 0.0003, |
| "step": 711 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09721463681048607, |
| "grad_norm": 0.037202075123786926, |
| "kl": 0.027284836760372855, |
| "learning_rate": 1.5157894736842108e-06, |
| "loss": 0.0002, |
| "step": 712 |
| }, |
| { |
| "clip_ratio": 0.0005803374951938167, |
| "completion_length": 134.25, |
| "epoch": 0.09735117422173675, |
| "grad_norm": 0.08356423676013947, |
| "kl": 0.028570065303938463, |
| "learning_rate": 1.5105263157894737e-06, |
| "loss": 0.0004, |
| "num_tokens": 2008914.0, |
| "reward": 0.887499988079071, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.887499988079071, |
| "step": 713 |
| }, |
| { |
| "clip_ratio": 0.00037202381645329297, |
| "epoch": 0.09748771163298744, |
| "grad_norm": 0.0837133526802063, |
| "kl": 0.029038649721769616, |
| "learning_rate": 1.5052631578947369e-06, |
| "loss": 0.0002, |
| "step": 714 |
| }, |
| { |
| "clip_ratio": 0.0002281021879753098, |
| "epoch": 0.09762424904423812, |
| "grad_norm": 0.0792241021990776, |
| "kl": 0.029093143355567008, |
| "learning_rate": 1.5e-06, |
| "loss": 0.0003, |
| "step": 715 |
| }, |
| { |
| "clip_ratio": 0.00018601190822664648, |
| "epoch": 0.0977607864554888, |
| "grad_norm": 0.07729717344045639, |
| "kl": 0.02941943213227205, |
| "learning_rate": 1.4947368421052632e-06, |
| "loss": -0.0002, |
| "step": 716 |
| }, |
| { |
| "clip_ratio": 0.0007804456472513266, |
| "completion_length": 154.4375, |
| "epoch": 0.09789732386673948, |
| "grad_norm": 0.08151241391897202, |
| "kl": 0.03156525699887425, |
| "learning_rate": 1.4894736842105264e-06, |
| "loss": 0.0002, |
| "num_tokens": 2021000.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 717 |
| }, |
| { |
| "clip_ratio": 0.00025540192291373387, |
| "epoch": 0.09803386127799017, |
| "grad_norm": 0.08159659057855606, |
| "kl": 0.0318286495457869, |
| "learning_rate": 1.4842105263157897e-06, |
| "loss": 0.0, |
| "step": 718 |
| }, |
| { |
| "clip_ratio": 0.0005171415978111327, |
| "epoch": 0.09817039868924085, |
| "grad_norm": 0.08123067766427994, |
| "kl": 0.031137815676629543, |
| "learning_rate": 1.478947368421053e-06, |
| "loss": -0.0001, |
| "step": 719 |
| }, |
| { |
| "clip_ratio": 0.0006710824818583205, |
| "epoch": 0.09830693610049153, |
| "grad_norm": 0.07915233820676804, |
| "kl": 0.031273662752937526, |
| "learning_rate": 1.4736842105263159e-06, |
| "loss": -0.0004, |
| "step": 720 |
| }, |
| { |
| "clip_ratio": 0.00026709403027780354, |
| "completion_length": 128.96875, |
| "epoch": 0.09844347351174222, |
| "grad_norm": 0.11582774668931961, |
| "kl": 0.03666126274038106, |
| "learning_rate": 1.468421052631579e-06, |
| "loss": 0.0003, |
| "num_tokens": 2032063.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.298653781414032, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 721 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "epoch": 0.0985800109229929, |
| "grad_norm": 0.11533697694540024, |
| "kl": 0.036149427643977106, |
| "learning_rate": 1.4631578947368422e-06, |
| "loss": 0.0, |
| "step": 722 |
| }, |
| { |
| "clip_ratio": 0.0005681818001903594, |
| "epoch": 0.09871654833424358, |
| "grad_norm": 0.11449582874774933, |
| "kl": 0.035863583092577755, |
| "learning_rate": 1.4578947368421053e-06, |
| "loss": -0.0001, |
| "step": 723 |
| }, |
| { |
| "clip_ratio": 0.0015406785241793841, |
| "epoch": 0.09885308574549427, |
| "grad_norm": 0.1062321737408638, |
| "kl": 0.03639277716865763, |
| "learning_rate": 1.4526315789473685e-06, |
| "loss": -0.0008, |
| "step": 724 |
| }, |
| { |
| "clip_ratio": 0.0002367424312978983, |
| "completion_length": 127.3125, |
| "epoch": 0.09898962315674495, |
| "grad_norm": 0.08646994829177856, |
| "kl": 0.031235893722623587, |
| "learning_rate": 1.4473684210526317e-06, |
| "loss": 0.0006, |
| "num_tokens": 2043593.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 725 |
| }, |
| { |
| "clip_ratio": 0.00031565656536258757, |
| "epoch": 0.09912616056799563, |
| "grad_norm": 0.08576333522796631, |
| "kl": 0.03090131061617285, |
| "learning_rate": 1.442105263157895e-06, |
| "loss": 0.0004, |
| "step": 726 |
| }, |
| { |
| "clip_ratio": 0.00015318627993110567, |
| "epoch": 0.09926269797924632, |
| "grad_norm": 0.08371236175298691, |
| "kl": 0.03052604233380407, |
| "learning_rate": 1.4368421052631582e-06, |
| "loss": 0.0002, |
| "step": 727 |
| }, |
| { |
| "clip_ratio": 0.0003327839804114774, |
| "epoch": 0.099399235390497, |
| "grad_norm": 0.08282700181007385, |
| "kl": 0.0304128993593622, |
| "learning_rate": 1.4315789473684212e-06, |
| "loss": -0.0003, |
| "step": 728 |
| }, |
| { |
| "clip_ratio": 0.0013521671062335372, |
| "completion_length": 135.6875, |
| "epoch": 0.09953577280174768, |
| "grad_norm": 0.120718814432621, |
| "kl": 0.030184058006852865, |
| "learning_rate": 1.4263157894736843e-06, |
| "loss": 0.0004, |
| "num_tokens": 2055143.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.3374999761581421, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 729 |
| }, |
| { |
| "clip_ratio": 0.00020559210679493845, |
| "epoch": 0.09967231021299836, |
| "grad_norm": 0.1224217414855957, |
| "kl": 0.030380245007108897, |
| "learning_rate": 1.4210526315789475e-06, |
| "loss": -0.0002, |
| "step": 730 |
| }, |
| { |
| "clip_ratio": 0.0008518250251654536, |
| "epoch": 0.09980884762424905, |
| "grad_norm": 0.12106145918369293, |
| "kl": 0.03084506734739989, |
| "learning_rate": 1.4157894736842107e-06, |
| "loss": -0.0005, |
| "step": 731 |
| }, |
| { |
| "clip_ratio": 0.0003453038807492703, |
| "epoch": 0.09994538503549973, |
| "grad_norm": 0.11929699033498764, |
| "kl": 0.03096500746323727, |
| "learning_rate": 1.4105263157894738e-06, |
| "loss": -0.001, |
| "step": 732 |
| }, |
| { |
| "clip_ratio": 0.0001220703125, |
| "completion_length": 123.625, |
| "epoch": 0.1000819224467504, |
| "grad_norm": 0.06932857632637024, |
| "kl": 0.03866506487247534, |
| "learning_rate": 1.405263157894737e-06, |
| "loss": 0.0003, |
| "num_tokens": 2066631.0, |
| "reward": 0.9437500238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.9437500238418579, |
| "step": 733 |
| }, |
| { |
| "clip_ratio": 0.0004617442318703979, |
| "epoch": 0.1002184598580011, |
| "grad_norm": 0.06737175583839417, |
| "kl": 0.0382203079352621, |
| "learning_rate": 1.4000000000000001e-06, |
| "loss": 0.0002, |
| "step": 734 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.10035499726925177, |
| "grad_norm": 0.06321597844362259, |
| "kl": 0.0391845365811605, |
| "learning_rate": 1.394736842105263e-06, |
| "loss": 0.0, |
| "step": 735 |
| }, |
| { |
| "clip_ratio": 0.0001220703125, |
| "epoch": 0.10049153468050245, |
| "grad_norm": 0.060581691563129425, |
| "kl": 0.039463951223297045, |
| "learning_rate": 1.3894736842105263e-06, |
| "loss": -0.0001, |
| "step": 736 |
| }, |
| { |
| "clip_ratio": 0.0004249223129590973, |
| "completion_length": 114.3125, |
| "epoch": 0.10062807209175315, |
| "grad_norm": 0.06796769797801971, |
| "kl": 0.03601354014244862, |
| "learning_rate": 1.3842105263157896e-06, |
| "loss": 0.0003, |
| "num_tokens": 2078197.0, |
| "reward": 0.859375, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.859375, |
| "step": 737 |
| }, |
| { |
| "clip_ratio": 0.00013185653369873762, |
| "epoch": 0.10076460950300382, |
| "grad_norm": 0.06704866886138916, |
| "kl": 0.036752572166733444, |
| "learning_rate": 1.3789473684210528e-06, |
| "loss": 0.0004, |
| "step": 738 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1009011469142545, |
| "grad_norm": 0.06749145686626434, |
| "kl": 0.036876781057799235, |
| "learning_rate": 1.373684210526316e-06, |
| "loss": 0.0003, |
| "step": 739 |
| }, |
| { |
| "clip_ratio": 0.00013185653369873762, |
| "epoch": 0.1010376843255052, |
| "grad_norm": 0.061885811388492584, |
| "kl": 0.036570252443198115, |
| "learning_rate": 1.3684210526315791e-06, |
| "loss": 0.0001, |
| "step": 740 |
| }, |
| { |
| "clip_ratio": 0.0006059740553610027, |
| "completion_length": 146.1875, |
| "epoch": 0.10117422173675587, |
| "grad_norm": 0.09527821838855743, |
| "kl": 0.032047255721408874, |
| "learning_rate": 1.3631578947368423e-06, |
| "loss": 0.0003, |
| "num_tokens": 2089871.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 741 |
| }, |
| { |
| "clip_ratio": 0.0011411394516471773, |
| "epoch": 0.10131075914800655, |
| "grad_norm": 0.09487205743789673, |
| "kl": 0.03190063979127444, |
| "learning_rate": 1.3578947368421052e-06, |
| "loss": 0.0002, |
| "step": 742 |
| }, |
| { |
| "clip_ratio": 0.0006642076768912375, |
| "epoch": 0.10144729655925723, |
| "grad_norm": 0.09328478574752808, |
| "kl": 0.03218909970019013, |
| "learning_rate": 1.3526315789473684e-06, |
| "loss": 0.0, |
| "step": 743 |
| }, |
| { |
| "clip_ratio": 0.0008540581329725683, |
| "epoch": 0.10158383397050792, |
| "grad_norm": 0.09467902779579163, |
| "kl": 0.03178860497428104, |
| "learning_rate": 1.3473684210526316e-06, |
| "loss": -0.0006, |
| "step": 744 |
| }, |
| { |
| "clip_ratio": 0.0014868229773128405, |
| "completion_length": 132.3125, |
| "epoch": 0.1017203713817586, |
| "grad_norm": 0.12321406602859497, |
| "kl": 0.03981124117854051, |
| "learning_rate": 1.342105263157895e-06, |
| "loss": 0.0006, |
| "num_tokens": 2101149.0, |
| "reward": 0.578125, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.578125, |
| "step": 745 |
| }, |
| { |
| "clip_ratio": 0.0005994522216496989, |
| "epoch": 0.10185690879300928, |
| "grad_norm": 0.11873092502355576, |
| "kl": 0.039661494083702564, |
| "learning_rate": 1.3368421052631581e-06, |
| "loss": 0.0002, |
| "step": 746 |
| }, |
| { |
| "clip_ratio": 0.0002943657018477097, |
| "epoch": 0.10199344620425997, |
| "grad_norm": 0.12101374566555023, |
| "kl": 0.039191018004203215, |
| "learning_rate": 1.3315789473684213e-06, |
| "loss": -0.0002, |
| "step": 747 |
| }, |
| { |
| "clip_ratio": 0.00041562868864275515, |
| "epoch": 0.10212998361551065, |
| "grad_norm": 0.11430719494819641, |
| "kl": 0.03897344155120663, |
| "learning_rate": 1.3263157894736844e-06, |
| "loss": -0.0006, |
| "step": 748 |
| }, |
| { |
| "clip_ratio": 0.0008153396483976394, |
| "completion_length": 129.5, |
| "epoch": 0.10226652102676133, |
| "grad_norm": 0.06621770560741425, |
| "kl": 0.030582898791180924, |
| "learning_rate": 1.3210526315789474e-06, |
| "loss": 0.0004, |
| "num_tokens": 2112669.0, |
| "reward": 0.831250011920929, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.831250011920929, |
| "step": 749 |
| }, |
| { |
| "clip_ratio": 0.0005412168393377215, |
| "epoch": 0.10240305843801202, |
| "grad_norm": 0.06624240428209305, |
| "kl": 0.03081196144921705, |
| "learning_rate": 1.3157894736842106e-06, |
| "loss": 0.0001, |
| "step": 750 |
| }, |
| { |
| "clip_ratio": 0.00026709403027780354, |
| "epoch": 0.1025395958492627, |
| "grad_norm": 0.0665903389453888, |
| "kl": 0.030628347245510668, |
| "learning_rate": 1.3105263157894737e-06, |
| "loss": 0.0, |
| "step": 751 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.10267613326051338, |
| "grad_norm": 0.0640382319688797, |
| "kl": 0.03100741034722887, |
| "learning_rate": 1.3052631578947369e-06, |
| "loss": -0.0002, |
| "step": 752 |
| }, |
| { |
| "clip_ratio": 0.0003129117249045521, |
| "completion_length": 142.8125, |
| "epoch": 0.10281267067176407, |
| "grad_norm": 0.10943469405174255, |
| "kl": 0.030879329569870606, |
| "learning_rate": 1.3e-06, |
| "loss": 0.0001, |
| "num_tokens": 2124911.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 753 |
| }, |
| { |
| "clip_ratio": 0.0004050493298564106, |
| "epoch": 0.10294920808301475, |
| "grad_norm": 0.11133232712745667, |
| "kl": 0.03102087226579897, |
| "learning_rate": 1.2947368421052634e-06, |
| "loss": -0.0002, |
| "step": 754 |
| }, |
| { |
| "clip_ratio": 0.0006491899548564106, |
| "epoch": 0.10308574549426543, |
| "grad_norm": 0.10690458118915558, |
| "kl": 0.030950611282605678, |
| "learning_rate": 1.2894736842105266e-06, |
| "loss": -0.0006, |
| "step": 755 |
| }, |
| { |
| "clip_ratio": 0.0012277475470909849, |
| "epoch": 0.1032222829055161, |
| "grad_norm": 0.10306268185377121, |
| "kl": 0.030929109954740852, |
| "learning_rate": 1.2842105263157895e-06, |
| "loss": -0.0008, |
| "step": 756 |
| }, |
| { |
| "clip_ratio": 0.0001849112450145185, |
| "completion_length": 136.25, |
| "epoch": 0.1033588203167668, |
| "grad_norm": 0.04018304869532585, |
| "kl": 0.03246450412552804, |
| "learning_rate": 1.2789473684210527e-06, |
| "loss": 0.0003, |
| "num_tokens": 2136827.0, |
| "reward": 0.6906249523162842, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.6906249523162842, |
| "step": 757 |
| }, |
| { |
| "clip_ratio": 0.00040654244367033243, |
| "epoch": 0.10349535772801748, |
| "grad_norm": 0.03952132910490036, |
| "kl": 0.033268949860939756, |
| "learning_rate": 1.2736842105263159e-06, |
| "loss": 0.0003, |
| "step": 758 |
| }, |
| { |
| "clip_ratio": 0.0001849112450145185, |
| "epoch": 0.10363189513926815, |
| "grad_norm": 0.039509255439043045, |
| "kl": 0.03303608257556334, |
| "learning_rate": 1.268421052631579e-06, |
| "loss": 0.0003, |
| "step": 759 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10376843255051885, |
| "grad_norm": 0.03916515037417412, |
| "kl": 0.032561466010520235, |
| "learning_rate": 1.2631578947368422e-06, |
| "loss": 0.0003, |
| "step": 760 |
| }, |
| { |
| "clip_ratio": 0.0006341315165627748, |
| "completion_length": 131.75, |
| "epoch": 0.10390496996176952, |
| "grad_norm": 0.08830566704273224, |
| "kl": 0.03105318936286494, |
| "learning_rate": 1.2578947368421054e-06, |
| "loss": 0.0002, |
| "num_tokens": 2148251.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 761 |
| }, |
| { |
| "clip_ratio": 0.0009561895130900666, |
| "epoch": 0.1040415073730202, |
| "grad_norm": 0.08864033222198486, |
| "kl": 0.03143153933342546, |
| "learning_rate": 1.2526315789473687e-06, |
| "loss": 0.0002, |
| "step": 762 |
| }, |
| { |
| "clip_ratio": 0.0012018284760415554, |
| "epoch": 0.1041780447842709, |
| "grad_norm": 0.08631853014230728, |
| "kl": 0.03134011913789436, |
| "learning_rate": 1.2473684210526317e-06, |
| "loss": 0.0, |
| "step": 763 |
| }, |
| { |
| "clip_ratio": 0.0006564522045664489, |
| "epoch": 0.10431458219552157, |
| "grad_norm": 0.08750317245721817, |
| "kl": 0.03160371968988329, |
| "learning_rate": 1.2421052631578948e-06, |
| "loss": -0.0002, |
| "step": 764 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 131.21875, |
| "epoch": 0.10445111960677225, |
| "grad_norm": 0.04507174715399742, |
| "kl": 0.03025782248005271, |
| "learning_rate": 1.236842105263158e-06, |
| "loss": 0.0003, |
| "num_tokens": 2159018.0, |
| "reward": 0.971875011920929, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.971875011920929, |
| "step": 765 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10458765701802294, |
| "grad_norm": 0.04543492570519447, |
| "kl": 0.03106310815201141, |
| "learning_rate": 1.2315789473684212e-06, |
| "loss": 0.0004, |
| "step": 766 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10472419442927362, |
| "grad_norm": 0.04544907435774803, |
| "kl": 0.030511865101289004, |
| "learning_rate": 1.2263157894736843e-06, |
| "loss": 0.0002, |
| "step": 767 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1048607318405243, |
| "grad_norm": 0.0443943589925766, |
| "kl": 0.02988862877828069, |
| "learning_rate": 1.2210526315789475e-06, |
| "loss": 0.0002, |
| "step": 768 |
| }, |
| { |
| "clip_ratio": 0.0007646681769983843, |
| "completion_length": 132.4375, |
| "epoch": 0.10499726925177498, |
| "grad_norm": 0.05918116122484207, |
| "kl": 0.031367348274216056, |
| "learning_rate": 1.2157894736842107e-06, |
| "loss": 0.0004, |
| "num_tokens": 2170364.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 769 |
| }, |
| { |
| "clip_ratio": 0.000615662953350693, |
| "epoch": 0.10513380666302567, |
| "grad_norm": 0.058712657541036606, |
| "kl": 0.031523832760285586, |
| "learning_rate": 1.2105263157894738e-06, |
| "loss": 0.0003, |
| "step": 770 |
| }, |
| { |
| "clip_ratio": 0.00041693481034599245, |
| "epoch": 0.10527034407427635, |
| "grad_norm": 0.05310717225074768, |
| "kl": 0.03125297068618238, |
| "learning_rate": 1.205263157894737e-06, |
| "loss": 0.0002, |
| "step": 771 |
| }, |
| { |
| "clip_ratio": 0.000615662953350693, |
| "epoch": 0.10540688148552703, |
| "grad_norm": 0.05317603796720505, |
| "kl": 0.031730295915622264, |
| "learning_rate": 1.2000000000000002e-06, |
| "loss": 0.0002, |
| "step": 772 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.84375, |
| "epoch": 0.10554341889677772, |
| "grad_norm": 0.08601640909910202, |
| "kl": 0.036299167724791914, |
| "learning_rate": 1.1947368421052633e-06, |
| "loss": 0.0004, |
| "num_tokens": 2181423.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 773 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1056799563080284, |
| "grad_norm": 0.08543814718723297, |
| "kl": 0.03631689946632832, |
| "learning_rate": 1.1894736842105265e-06, |
| "loss": 0.0001, |
| "step": 774 |
| }, |
| { |
| "clip_ratio": 0.0004579307569656521, |
| "epoch": 0.10581649371927908, |
| "grad_norm": 0.0862925574183464, |
| "kl": 0.03603148739784956, |
| "learning_rate": 1.1842105263157894e-06, |
| "loss": 0.0001, |
| "step": 775 |
| }, |
| { |
| "clip_ratio": 0.00023148147738538682, |
| "epoch": 0.10595303113052977, |
| "grad_norm": 0.08512796461582184, |
| "kl": 0.03633074296521954, |
| "learning_rate": 1.1789473684210526e-06, |
| "loss": -0.0002, |
| "step": 776 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 120.15625, |
| "epoch": 0.10608956854178045, |
| "grad_norm": 0.04055812582373619, |
| "kl": 0.03543747233925387, |
| "learning_rate": 1.173684210526316e-06, |
| "loss": 0.0003, |
| "num_tokens": 2192324.0, |
| "reward": 0.578125, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.578125, |
| "step": 777 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10622610595303113, |
| "grad_norm": 0.040661316365003586, |
| "kl": 0.035565093508921564, |
| "learning_rate": 1.1684210526315791e-06, |
| "loss": 0.0002, |
| "step": 778 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10636264336428182, |
| "grad_norm": 0.04156764596700668, |
| "kl": 0.0351781653589569, |
| "learning_rate": 1.163157894736842e-06, |
| "loss": 0.0002, |
| "step": 779 |
| }, |
| { |
| "clip_ratio": 0.00028669723542407155, |
| "epoch": 0.1064991807755325, |
| "grad_norm": 0.04101695492863655, |
| "kl": 0.03485118242679164, |
| "learning_rate": 1.1578947368421053e-06, |
| "loss": 0.0002, |
| "step": 780 |
| }, |
| { |
| "clip_ratio": 0.0012449884961824864, |
| "completion_length": 144.90625, |
| "epoch": 0.10663571818678318, |
| "grad_norm": 0.09758393466472626, |
| "kl": 0.037451187847182155, |
| "learning_rate": 1.1526315789473686e-06, |
| "loss": 0.0005, |
| "num_tokens": 2204517.0, |
| "reward": 0.3812500238418579, |
| "reward_std": 0.1948557198047638, |
| "rewards/reward_fn": 0.3812500238418579, |
| "step": 781 |
| }, |
| { |
| "clip_ratio": 0.0007635208748979494, |
| "epoch": 0.10677225559803386, |
| "grad_norm": 0.09678144752979279, |
| "kl": 0.03737691760761663, |
| "learning_rate": 1.1473684210526316e-06, |
| "loss": 0.0005, |
| "step": 782 |
| }, |
| { |
| "clip_ratio": 0.00034340660204179585, |
| "epoch": 0.10690879300928455, |
| "grad_norm": 0.0968538299202919, |
| "kl": 0.03713806488667615, |
| "learning_rate": 1.1421052631578947e-06, |
| "loss": 0.0003, |
| "step": 783 |
| }, |
| { |
| "clip_ratio": 0.000750611608964391, |
| "epoch": 0.10704533042053523, |
| "grad_norm": 0.09521595388650894, |
| "kl": 0.037653202831279486, |
| "learning_rate": 1.136842105263158e-06, |
| "loss": 0.0, |
| "step": 784 |
| }, |
| { |
| "clip_ratio": 0.0013667781895492226, |
| "completion_length": 131.09375, |
| "epoch": 0.1071818678317859, |
| "grad_norm": 0.145091250538826, |
| "kl": 0.03401468286756426, |
| "learning_rate": 1.1315789473684213e-06, |
| "loss": 0.0001, |
| "num_tokens": 2216008.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 785 |
| }, |
| { |
| "clip_ratio": 0.0008191931410692632, |
| "epoch": 0.1073184052430366, |
| "grad_norm": 0.14306364953517914, |
| "kl": 0.03373881638981402, |
| "learning_rate": 1.1263157894736842e-06, |
| "loss": -0.0003, |
| "step": 786 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10745494265428727, |
| "grad_norm": 0.1369042545557022, |
| "kl": 0.03430832191952504, |
| "learning_rate": 1.1210526315789474e-06, |
| "loss": -0.0004, |
| "step": 787 |
| }, |
| { |
| "clip_ratio": 0.0005872874171473086, |
| "epoch": 0.10759148006553795, |
| "grad_norm": 0.1372627466917038, |
| "kl": 0.03363573501701467, |
| "learning_rate": 1.1157894736842106e-06, |
| "loss": -0.0013, |
| "step": 788 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 128.40625, |
| "epoch": 0.10772801747678865, |
| "grad_norm": 0.05001708120107651, |
| "kl": 0.03717817255528644, |
| "learning_rate": 1.110526315789474e-06, |
| "loss": 0.0003, |
| "num_tokens": 2226477.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 789 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10786455488803932, |
| "grad_norm": 0.047382958233356476, |
| "kl": 0.0376498686382547, |
| "learning_rate": 1.1052631578947369e-06, |
| "loss": 0.0003, |
| "step": 790 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10800109229929, |
| "grad_norm": 0.048519086092710495, |
| "kl": 0.03672667942009866, |
| "learning_rate": 1.1e-06, |
| "loss": 0.0003, |
| "step": 791 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1081376297105407, |
| "grad_norm": 0.04772576689720154, |
| "kl": 0.03647302172612399, |
| "learning_rate": 1.0947368421052632e-06, |
| "loss": 0.0002, |
| "step": 792 |
| }, |
| { |
| "clip_ratio": 0.0006341192492982373, |
| "completion_length": 131.53125, |
| "epoch": 0.10827416712179137, |
| "grad_norm": 0.08512444794178009, |
| "kl": 0.028109061560826376, |
| "learning_rate": 1.0894736842105264e-06, |
| "loss": 0.0001, |
| "num_tokens": 2238058.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 793 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10841070453304205, |
| "grad_norm": 0.08351006358861923, |
| "kl": 0.028101337986299768, |
| "learning_rate": 1.0842105263157895e-06, |
| "loss": 0.0001, |
| "step": 794 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10854724194429273, |
| "grad_norm": 0.08248769491910934, |
| "kl": 0.028218604769790545, |
| "learning_rate": 1.0789473684210527e-06, |
| "loss": -0.0001, |
| "step": 795 |
| }, |
| { |
| "clip_ratio": 0.0003955696302000433, |
| "epoch": 0.10868377935554342, |
| "grad_norm": 0.08255155384540558, |
| "kl": 0.028063812758773565, |
| "learning_rate": 1.0736842105263159e-06, |
| "loss": -0.0003, |
| "step": 796 |
| }, |
| { |
| "clip_ratio": 0.00044361942855175585, |
| "completion_length": 125.34375, |
| "epoch": 0.1088203167667941, |
| "grad_norm": 0.09224527329206467, |
| "kl": 0.033332623599562794, |
| "learning_rate": 1.068421052631579e-06, |
| "loss": 0.0005, |
| "num_tokens": 2250205.0, |
| "reward": 0.6906249523162842, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.6906249523162842, |
| "step": 797 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10895685417804478, |
| "grad_norm": 0.09290765970945358, |
| "kl": 0.033241317549254745, |
| "learning_rate": 1.0631578947368422e-06, |
| "loss": 0.0003, |
| "step": 798 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10909339158929547, |
| "grad_norm": 0.09002580493688583, |
| "kl": 0.03237891127355397, |
| "learning_rate": 1.0578947368421054e-06, |
| "loss": 0.0, |
| "step": 799 |
| }, |
| { |
| "clip_ratio": 0.000240384615608491, |
| "epoch": 0.10922992900054615, |
| "grad_norm": 0.09006845206022263, |
| "kl": 0.0326729342341423, |
| "learning_rate": 1.0526315789473685e-06, |
| "loss": -0.0002, |
| "step": 800 |
| }, |
| { |
| "clip_ratio": 0.0003324467979837209, |
| "completion_length": 115.53125, |
| "epoch": 0.10936646641179683, |
| "grad_norm": 0.05152610316872597, |
| "kl": 0.031394801451824605, |
| "learning_rate": 1.0473684210526317e-06, |
| "loss": 0.0005, |
| "num_tokens": 2260810.0, |
| "reward": 0.71875, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.71875, |
| "step": 801 |
| }, |
| { |
| "clip_ratio": 0.0003324467979837209, |
| "epoch": 0.10950300382304752, |
| "grad_norm": 0.051167577505111694, |
| "kl": 0.03137708600843325, |
| "learning_rate": 1.0421052631578949e-06, |
| "loss": 0.0004, |
| "step": 802 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1096395412342982, |
| "grad_norm": 0.05100513994693756, |
| "kl": 0.03141244366997853, |
| "learning_rate": 1.036842105263158e-06, |
| "loss": 0.0003, |
| "step": 803 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10977607864554888, |
| "grad_norm": 0.0502689927816391, |
| "kl": 0.03105094050988555, |
| "learning_rate": 1.0315789473684212e-06, |
| "loss": 0.0002, |
| "step": 804 |
| }, |
| { |
| "clip_ratio": 0.0001990445889532566, |
| "completion_length": 132.75, |
| "epoch": 0.10991261605679957, |
| "grad_norm": 0.06489317119121552, |
| "kl": 0.034109927713871, |
| "learning_rate": 1.0263157894736843e-06, |
| "loss": 0.0004, |
| "num_tokens": 2272318.0, |
| "reward": 0.831250011920929, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.831250011920929, |
| "step": 805 |
| }, |
| { |
| "clip_ratio": 0.0004087761335540563, |
| "epoch": 0.11004915346805025, |
| "grad_norm": 0.058789148926734924, |
| "kl": 0.03437470211065374, |
| "learning_rate": 1.0210526315789475e-06, |
| "loss": 0.0005, |
| "step": 806 |
| }, |
| { |
| "clip_ratio": 0.00020973154460079968, |
| "epoch": 0.11018569087930093, |
| "grad_norm": 0.06509160250425339, |
| "kl": 0.034245212213136256, |
| "learning_rate": 1.0157894736842105e-06, |
| "loss": 0.0004, |
| "step": 807 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1103222282905516, |
| "grad_norm": 0.0659160166978836, |
| "kl": 0.03385831121704541, |
| "learning_rate": 1.0105263157894738e-06, |
| "loss": 0.0002, |
| "step": 808 |
| }, |
| { |
| "clip_ratio": 0.0010227008606307209, |
| "completion_length": 126.8125, |
| "epoch": 0.1104587657018023, |
| "grad_norm": 0.10646427422761917, |
| "kl": 0.03514612466096878, |
| "learning_rate": 1.005263157894737e-06, |
| "loss": 0.0004, |
| "num_tokens": 2283260.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 809 |
| }, |
| { |
| "clip_ratio": 0.0009959485905710608, |
| "epoch": 0.11059530311305298, |
| "grad_norm": 0.10746331512928009, |
| "kl": 0.03498730491264723, |
| "learning_rate": 1.0000000000000002e-06, |
| "loss": 0.0004, |
| "step": 810 |
| }, |
| { |
| "clip_ratio": 0.001203794716275297, |
| "epoch": 0.11073184052430365, |
| "grad_norm": 0.1056860014796257, |
| "kl": 0.03553603668115102, |
| "learning_rate": 9.947368421052631e-07, |
| "loss": 0.0003, |
| "step": 811 |
| }, |
| { |
| "clip_ratio": 0.0003511235991027206, |
| "epoch": 0.11086837793555435, |
| "grad_norm": 0.1063576340675354, |
| "kl": 0.034650685905944556, |
| "learning_rate": 9.894736842105265e-07, |
| "loss": -0.0003, |
| "step": 812 |
| }, |
| { |
| "clip_ratio": 0.0005857369105797261, |
| "completion_length": 108.96875, |
| "epoch": 0.11100491534680502, |
| "grad_norm": 0.13227391242980957, |
| "kl": 0.03337117409682833, |
| "learning_rate": 9.842105263157897e-07, |
| "loss": 0.0005, |
| "num_tokens": 2294083.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 813 |
| }, |
| { |
| "clip_ratio": 0.0002460629912093282, |
| "epoch": 0.1111414527580557, |
| "grad_norm": 0.12803682684898376, |
| "kl": 0.032956252398435026, |
| "learning_rate": 9.789473684210526e-07, |
| "loss": -0.0, |
| "step": 814 |
| }, |
| { |
| "clip_ratio": 0.0008287565433420241, |
| "epoch": 0.1112779901693064, |
| "grad_norm": 0.12848548591136932, |
| "kl": 0.032824013265781105, |
| "learning_rate": 9.736842105263158e-07, |
| "loss": -0.0, |
| "step": 815 |
| }, |
| { |
| "clip_ratio": 0.0002460629912093282, |
| "epoch": 0.11141452758055707, |
| "grad_norm": 0.11636755615472794, |
| "kl": 0.032896427903324366, |
| "learning_rate": 9.68421052631579e-07, |
| "loss": -0.0004, |
| "step": 816 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 119.65625, |
| "epoch": 0.11155106499180775, |
| "grad_norm": 0.0969424620270729, |
| "kl": 0.03230866280500777, |
| "learning_rate": 9.631578947368423e-07, |
| "loss": 0.0001, |
| "num_tokens": 2304824.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 817 |
| }, |
| { |
| "clip_ratio": 0.00019778481510002166, |
| "epoch": 0.11168760240305844, |
| "grad_norm": 0.09678807854652405, |
| "kl": 0.032764478790340945, |
| "learning_rate": 9.578947368421053e-07, |
| "loss": 0.0002, |
| "step": 818 |
| }, |
| { |
| "clip_ratio": 0.00047374211135320365, |
| "epoch": 0.11182413981430912, |
| "grad_norm": 0.09999542683362961, |
| "kl": 0.03256647096714005, |
| "learning_rate": 9.526315789473685e-07, |
| "loss": 0.0002, |
| "step": 819 |
| }, |
| { |
| "clip_ratio": 0.00047374211135320365, |
| "epoch": 0.1119606772255598, |
| "grad_norm": 0.0940428152680397, |
| "kl": 0.03249253722606227, |
| "learning_rate": 9.473684210526317e-07, |
| "loss": -0.0002, |
| "step": 820 |
| }, |
| { |
| "clip_ratio": 0.0004139072843827307, |
| "completion_length": 129.6875, |
| "epoch": 0.11209721463681048, |
| "grad_norm": 0.08134789764881134, |
| "kl": 0.04074790241429582, |
| "learning_rate": 9.421052631578948e-07, |
| "loss": 0.0004, |
| "num_tokens": 2315998.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 821 |
| }, |
| { |
| "clip_ratio": 0.0006293771584751084, |
| "epoch": 0.11223375204806117, |
| "grad_norm": 0.07964572310447693, |
| "kl": 0.040875735925510526, |
| "learning_rate": 9.368421052631579e-07, |
| "loss": 0.0005, |
| "step": 822 |
| }, |
| { |
| "clip_ratio": 0.00020695364219136536, |
| "epoch": 0.11237028945931185, |
| "grad_norm": 0.07946870476007462, |
| "kl": 0.0408237244701013, |
| "learning_rate": 9.315789473684212e-07, |
| "loss": 0.0001, |
| "step": 823 |
| }, |
| { |
| "clip_ratio": 0.00020695364219136536, |
| "epoch": 0.11250682687056253, |
| "grad_norm": 0.07891543954610825, |
| "kl": 0.041721748071722686, |
| "learning_rate": 9.263157894736844e-07, |
| "loss": -0.0, |
| "step": 824 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 125.1875, |
| "epoch": 0.11264336428181322, |
| "grad_norm": 0.04926425591111183, |
| "kl": 0.03286067905719392, |
| "learning_rate": 9.210526315789474e-07, |
| "loss": 0.0004, |
| "num_tokens": 2327484.0, |
| "reward": 0.9437500238418579, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.9437500238418579, |
| "step": 825 |
| }, |
| { |
| "clip_ratio": 0.00040524803625885397, |
| "epoch": 0.1127799016930639, |
| "grad_norm": 0.04810812696814537, |
| "kl": 0.03193527160328813, |
| "learning_rate": 9.157894736842106e-07, |
| "loss": 0.0003, |
| "step": 826 |
| }, |
| { |
| "clip_ratio": 0.00042229730752296746, |
| "epoch": 0.11291643910431458, |
| "grad_norm": 0.04899256303906441, |
| "kl": 0.03242765832692385, |
| "learning_rate": 9.105263157894737e-07, |
| "loss": 0.0003, |
| "step": 827 |
| }, |
| { |
| "clip_ratio": 0.0003544972714735195, |
| "epoch": 0.11305297651556527, |
| "grad_norm": 0.049238696694374084, |
| "kl": 0.03237058556987904, |
| "learning_rate": 9.05263157894737e-07, |
| "loss": 0.0003, |
| "step": 828 |
| }, |
| { |
| "clip_ratio": 0.0005081300623714924, |
| "completion_length": 125.4375, |
| "epoch": 0.11318951392681595, |
| "grad_norm": 0.13235945999622345, |
| "kl": 0.03304180080885999, |
| "learning_rate": 9.000000000000001e-07, |
| "loss": 0.0007, |
| "num_tokens": 2338158.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 829 |
| }, |
| { |
| "clip_ratio": 0.0011491246259538457, |
| "epoch": 0.11332605133806663, |
| "grad_norm": 0.13015788793563843, |
| "kl": 0.033487798733403906, |
| "learning_rate": 8.947368421052632e-07, |
| "loss": 0.0004, |
| "step": 830 |
| }, |
| { |
| "clip_ratio": 0.00040650405571796, |
| "epoch": 0.11346258874931732, |
| "grad_norm": 0.12753689289093018, |
| "kl": 0.033373525278875604, |
| "learning_rate": 8.894736842105264e-07, |
| "loss": 0.0001, |
| "step": 831 |
| }, |
| { |
| "clip_ratio": 0.00022163119865581393, |
| "epoch": 0.113599126160568, |
| "grad_norm": 0.11997953057289124, |
| "kl": 0.03349483574857004, |
| "learning_rate": 8.842105263157895e-07, |
| "loss": -0.0002, |
| "step": 832 |
| }, |
| { |
| "clip_ratio": 0.0005183477042010054, |
| "completion_length": 131.71875, |
| "epoch": 0.11373566357181868, |
| "grad_norm": 0.1369657963514328, |
| "kl": 0.03463814881979488, |
| "learning_rate": 8.789473684210527e-07, |
| "loss": 0.0006, |
| "num_tokens": 2349773.0, |
| "reward": 0.6343750357627869, |
| "reward_std": 0.34620189666748047, |
| "rewards/reward_fn": 0.6343750357627869, |
| "step": 833 |
| }, |
| { |
| "clip_ratio": 0.001799030287656933, |
| "epoch": 0.11387220098306935, |
| "grad_norm": 0.13694268465042114, |
| "kl": 0.035131270095007494, |
| "learning_rate": 8.736842105263159e-07, |
| "loss": 0.0007, |
| "step": 834 |
| }, |
| { |
| "clip_ratio": 0.00014400921645574272, |
| "epoch": 0.11400873839432005, |
| "grad_norm": 0.12898842990398407, |
| "kl": 0.034667212778003886, |
| "learning_rate": 8.68421052631579e-07, |
| "loss": 0.0001, |
| "step": 835 |
| }, |
| { |
| "clip_ratio": 0.0007648299215361476, |
| "epoch": 0.11414527580557073, |
| "grad_norm": 0.1323985755443573, |
| "kl": 0.03442463994724676, |
| "learning_rate": 8.631578947368421e-07, |
| "loss": -0.0004, |
| "step": 836 |
| }, |
| { |
| "clip_ratio": 0.00035511364694684744, |
| "completion_length": 134.71875, |
| "epoch": 0.1142818132168214, |
| "grad_norm": 0.07983018457889557, |
| "kl": 0.03501651559781749, |
| "learning_rate": 8.578947368421054e-07, |
| "loss": 0.0003, |
| "num_tokens": 2361200.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 837 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1144183506280721, |
| "grad_norm": 0.07975803315639496, |
| "kl": 0.034936482581542805, |
| "learning_rate": 8.526315789473685e-07, |
| "loss": 0.0003, |
| "step": 838 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11455488803932277, |
| "grad_norm": 0.07610788196325302, |
| "kl": 0.03467000252567232, |
| "learning_rate": 8.473684210526316e-07, |
| "loss": 0.0001, |
| "step": 839 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11469142545057345, |
| "grad_norm": 0.07802704721689224, |
| "kl": 0.03485574647493195, |
| "learning_rate": 8.421052631578948e-07, |
| "loss": -0.0, |
| "step": 840 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 133.0625, |
| "epoch": 0.11482796286182415, |
| "grad_norm": 0.04737474024295807, |
| "kl": 0.0320931714377366, |
| "learning_rate": 8.36842105263158e-07, |
| "loss": 0.0003, |
| "num_tokens": 2372558.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 841 |
| }, |
| { |
| "clip_ratio": 0.0001220703125, |
| "epoch": 0.11496450027307482, |
| "grad_norm": 0.0413704551756382, |
| "kl": 0.03146939189173281, |
| "learning_rate": 8.315789473684212e-07, |
| "loss": 0.0002, |
| "step": 842 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1151010376843255, |
| "grad_norm": 0.041506968438625336, |
| "kl": 0.03132388353697024, |
| "learning_rate": 8.263157894736843e-07, |
| "loss": 0.0002, |
| "step": 843 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1152375750955762, |
| "grad_norm": 0.038393907248973846, |
| "kl": 0.031428944814251736, |
| "learning_rate": 8.210526315789474e-07, |
| "loss": 0.0001, |
| "step": 844 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.46875, |
| "epoch": 0.11537411250682687, |
| "grad_norm": 0.062188491225242615, |
| "kl": 0.036099575518164784, |
| "learning_rate": 8.157894736842106e-07, |
| "loss": 0.0002, |
| "num_tokens": 2384597.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 845 |
| }, |
| { |
| "clip_ratio": 0.0002367424312978983, |
| "epoch": 0.11551064991807755, |
| "grad_norm": 0.059262264519929886, |
| "kl": 0.035657460655784234, |
| "learning_rate": 8.105263157894736e-07, |
| "loss": 0.0001, |
| "step": 846 |
| }, |
| { |
| "clip_ratio": 0.0002367424312978983, |
| "epoch": 0.11564718732932823, |
| "grad_norm": 0.05964317545294762, |
| "kl": 0.03621940390439704, |
| "learning_rate": 8.052631578947369e-07, |
| "loss": 0.0002, |
| "step": 847 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11578372474057892, |
| "grad_norm": 0.0571606270968914, |
| "kl": 0.03622490944690071, |
| "learning_rate": 8.000000000000001e-07, |
| "loss": 0.0001, |
| "step": 848 |
| }, |
| { |
| "clip_ratio": 0.0009445661416975781, |
| "completion_length": 135.84375, |
| "epoch": 0.1159202621518296, |
| "grad_norm": 0.07988615334033966, |
| "kl": 0.03682940514408983, |
| "learning_rate": 7.947368421052632e-07, |
| "loss": 0.0005, |
| "num_tokens": 2395576.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 849 |
| }, |
| { |
| "clip_ratio": 0.000547244620975107, |
| "epoch": 0.11605679956308028, |
| "grad_norm": 0.08060329407453537, |
| "kl": 0.03640499952598475, |
| "learning_rate": 7.894736842105263e-07, |
| "loss": 0.0004, |
| "step": 850 |
| }, |
| { |
| "clip_ratio": 0.0003555268340278417, |
| "epoch": 0.11619333697433097, |
| "grad_norm": 0.07949917018413544, |
| "kl": 0.03623723401688039, |
| "learning_rate": 7.842105263157896e-07, |
| "loss": 0.0002, |
| "step": 851 |
| }, |
| { |
| "clip_ratio": 0.0009964463242795318, |
| "epoch": 0.11632987438558165, |
| "grad_norm": 0.07702900469303131, |
| "kl": 0.036871126416372135, |
| "learning_rate": 7.789473684210527e-07, |
| "loss": 0.0003, |
| "step": 852 |
| }, |
| { |
| "clip_ratio": 0.0005859898083144799, |
| "completion_length": 120.15625, |
| "epoch": 0.11646641179683233, |
| "grad_norm": 0.0983206108212471, |
| "kl": 0.03288867976516485, |
| "learning_rate": 7.736842105263158e-07, |
| "loss": 0.0004, |
| "num_tokens": 2406365.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 853 |
| }, |
| { |
| "clip_ratio": 0.0006730413442710415, |
| "epoch": 0.11660294920808302, |
| "grad_norm": 0.09928011149168015, |
| "kl": 0.032499166671186686, |
| "learning_rate": 7.684210526315789e-07, |
| "loss": 0.0002, |
| "step": 854 |
| }, |
| { |
| "clip_ratio": 0.0004937214544042945, |
| "epoch": 0.1167394866193337, |
| "grad_norm": 0.0939321517944336, |
| "kl": 0.03273730678483844, |
| "learning_rate": 7.631578947368422e-07, |
| "loss": -0.0, |
| "step": 855 |
| }, |
| { |
| "clip_ratio": 0.00020833333837799728, |
| "epoch": 0.11687602403058438, |
| "grad_norm": 0.09672950953245163, |
| "kl": 0.03311119574937038, |
| "learning_rate": 7.578947368421054e-07, |
| "loss": -0.0002, |
| "step": 856 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "completion_length": 133.40625, |
| "epoch": 0.11701256144183507, |
| "grad_norm": 0.056489668786525726, |
| "kl": 0.030313314258819446, |
| "learning_rate": 7.526315789473684e-07, |
| "loss": 0.0004, |
| "num_tokens": 2417958.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 857 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11714909885308575, |
| "grad_norm": 0.0562896803021431, |
| "kl": 0.03055126141407527, |
| "learning_rate": 7.473684210526316e-07, |
| "loss": 0.0004, |
| "step": 858 |
| }, |
| { |
| "clip_ratio": 0.0003907771751983091, |
| "epoch": 0.11728563626433643, |
| "grad_norm": 0.05616592615842819, |
| "kl": 0.030165656498866156, |
| "learning_rate": 7.421052631578949e-07, |
| "loss": 0.0004, |
| "step": 859 |
| }, |
| { |
| "clip_ratio": 0.0004477034672163427, |
| "epoch": 0.1174221736755871, |
| "grad_norm": 0.05474083498120308, |
| "kl": 0.030640476441476494, |
| "learning_rate": 7.368421052631579e-07, |
| "loss": 0.0002, |
| "step": 860 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 129.6875, |
| "epoch": 0.1175587110868378, |
| "grad_norm": 0.10344516485929489, |
| "kl": 0.037577818031422794, |
| "learning_rate": 7.315789473684211e-07, |
| "loss": 0.0003, |
| "num_tokens": 2429480.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 861 |
| }, |
| { |
| "clip_ratio": 0.00020032051543239504, |
| "epoch": 0.11769524849808848, |
| "grad_norm": 0.10587334632873535, |
| "kl": 0.03769170871237293, |
| "learning_rate": 7.263157894736843e-07, |
| "loss": 0.0002, |
| "step": 862 |
| }, |
| { |
| "clip_ratio": 0.0004006410308647901, |
| "epoch": 0.11783178590933915, |
| "grad_norm": 0.10316218435764313, |
| "kl": 0.037826200015842915, |
| "learning_rate": 7.210526315789475e-07, |
| "loss": 0.0001, |
| "step": 863 |
| }, |
| { |
| "clip_ratio": 0.0004485645913518965, |
| "epoch": 0.11796832332058985, |
| "grad_norm": 0.1059485375881195, |
| "kl": 0.037548283056821674, |
| "learning_rate": 7.157894736842106e-07, |
| "loss": -0.0002, |
| "step": 864 |
| }, |
| { |
| "clip_ratio": 0.00033602150506339967, |
| "completion_length": 132.25, |
| "epoch": 0.11810486073184052, |
| "grad_norm": 0.09281036257743835, |
| "kl": 0.03125832715886645, |
| "learning_rate": 7.105263157894737e-07, |
| "loss": 0.0002, |
| "num_tokens": 2441072.0, |
| "reward": 0.46562501788139343, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.46562501788139343, |
| "step": 865 |
| }, |
| { |
| "clip_ratio": 0.0005000000237487257, |
| "epoch": 0.1182413981430912, |
| "grad_norm": 0.09430094808340073, |
| "kl": 0.03168348016333766, |
| "learning_rate": 7.052631578947369e-07, |
| "loss": 0.0003, |
| "step": 866 |
| }, |
| { |
| "clip_ratio": 0.0002500000118743628, |
| "epoch": 0.1183779355543419, |
| "grad_norm": 0.0902150422334671, |
| "kl": 0.03158555779373273, |
| "learning_rate": 7.000000000000001e-07, |
| "loss": 0.0002, |
| "step": 867 |
| }, |
| { |
| "clip_ratio": 0.00019654087373055518, |
| "epoch": 0.11851447296559257, |
| "grad_norm": 0.09070498496294022, |
| "kl": 0.031180568737909198, |
| "learning_rate": 6.947368421052631e-07, |
| "loss": -0.0002, |
| "step": 868 |
| }, |
| { |
| "clip_ratio": 0.00033602150506339967, |
| "completion_length": 110.90625, |
| "epoch": 0.11865101037684325, |
| "grad_norm": 0.06536060571670532, |
| "kl": 0.031654038379201666, |
| "learning_rate": 6.894736842105264e-07, |
| "loss": 0.0004, |
| "num_tokens": 2451289.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 869 |
| }, |
| { |
| "clip_ratio": 0.0010303832968929783, |
| "epoch": 0.11878754778809394, |
| "grad_norm": 0.06535768508911133, |
| "kl": 0.03135678684338927, |
| "learning_rate": 6.842105263157896e-07, |
| "loss": 0.0005, |
| "step": 870 |
| }, |
| { |
| "clip_ratio": 0.0011440572561696172, |
| "epoch": 0.11892408519934462, |
| "grad_norm": 0.06512617319822311, |
| "kl": 0.031891329621430486, |
| "learning_rate": 6.789473684210526e-07, |
| "loss": 0.0004, |
| "step": 871 |
| }, |
| { |
| "clip_ratio": 0.001402321708155796, |
| "epoch": 0.1190606226105953, |
| "grad_norm": 0.06582646071910858, |
| "kl": 0.03158884108415805, |
| "learning_rate": 6.736842105263158e-07, |
| "loss": 0.0004, |
| "step": 872 |
| }, |
| { |
| "clip_ratio": 0.00038580247201025486, |
| "completion_length": 111.09375, |
| "epoch": 0.11919716002184598, |
| "grad_norm": 0.0955490916967392, |
| "kl": 0.038142890349263325, |
| "learning_rate": 6.684210526315791e-07, |
| "loss": 0.0004, |
| "num_tokens": 2462664.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 873 |
| }, |
| { |
| "clip_ratio": 0.000405844155466184, |
| "epoch": 0.11933369743309667, |
| "grad_norm": 0.09661541134119034, |
| "kl": 0.037784790445584804, |
| "learning_rate": 6.631578947368422e-07, |
| "loss": 0.0003, |
| "step": 874 |
| }, |
| { |
| "clip_ratio": 0.0006363249995047227, |
| "epoch": 0.11947023484434735, |
| "grad_norm": 0.0970974862575531, |
| "kl": 0.03744145468226634, |
| "learning_rate": 6.578947368421053e-07, |
| "loss": 0.0003, |
| "step": 875 |
| }, |
| { |
| "clip_ratio": 0.0003878333227476105, |
| "epoch": 0.11960677225559803, |
| "grad_norm": 0.09769237041473389, |
| "kl": 0.03685220205807127, |
| "learning_rate": 6.526315789473684e-07, |
| "loss": 0.0001, |
| "step": 876 |
| }, |
| { |
| "clip_ratio": 0.0003955696302000433, |
| "completion_length": 116.625, |
| "epoch": 0.11974330966684872, |
| "grad_norm": 0.08843936026096344, |
| "kl": 0.03245319478446618, |
| "learning_rate": 6.473684210526317e-07, |
| "loss": 0.0003, |
| "num_tokens": 2473228.0, |
| "reward": 0.859375, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.859375, |
| "step": 877 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1198798470780994, |
| "grad_norm": 0.08856622129678726, |
| "kl": 0.032318223151378334, |
| "learning_rate": 6.421052631578948e-07, |
| "loss": 0.0002, |
| "step": 878 |
| }, |
| { |
| "clip_ratio": 0.00016191709437407553, |
| "epoch": 0.12001638448935008, |
| "grad_norm": 0.08873509615659714, |
| "kl": 0.03196106932591647, |
| "learning_rate": 6.368421052631579e-07, |
| "loss": 0.0, |
| "step": 879 |
| }, |
| { |
| "clip_ratio": 0.0003955696302000433, |
| "epoch": 0.12015292190060077, |
| "grad_norm": 0.08736385405063629, |
| "kl": 0.03180155347217806, |
| "learning_rate": 6.315789473684211e-07, |
| "loss": 0.0002, |
| "step": 880 |
| }, |
| { |
| "clip_ratio": 0.00028669723542407155, |
| "completion_length": 144.90625, |
| "epoch": 0.12028945931185145, |
| "grad_norm": 0.0820450633764267, |
| "kl": 0.037947153381537646, |
| "learning_rate": 6.263157894736844e-07, |
| "loss": 0.0004, |
| "num_tokens": 2485273.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 881 |
| }, |
| { |
| "clip_ratio": 0.0006903430767124519, |
| "epoch": 0.12042599672310213, |
| "grad_norm": 0.08431774377822876, |
| "kl": 0.03700205200584605, |
| "learning_rate": 6.210526315789474e-07, |
| "loss": 0.0005, |
| "step": 882 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12056253413435282, |
| "grad_norm": 0.08097940683364868, |
| "kl": 0.03734048001933843, |
| "learning_rate": 6.157894736842106e-07, |
| "loss": 0.0002, |
| "step": 883 |
| }, |
| { |
| "clip_ratio": 0.00020833333837799728, |
| "epoch": 0.1206990715456035, |
| "grad_norm": 0.0801195502281189, |
| "kl": 0.037502437888178974, |
| "learning_rate": 6.105263157894738e-07, |
| "loss": 0.0002, |
| "step": 884 |
| }, |
| { |
| "clip_ratio": 0.0008994654635898769, |
| "completion_length": 114.625, |
| "epoch": 0.12083560895685418, |
| "grad_norm": 0.09578239172697067, |
| "kl": 0.03379804195719771, |
| "learning_rate": 6.052631578947369e-07, |
| "loss": 0.0004, |
| "num_tokens": 2496109.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 885 |
| }, |
| { |
| "clip_ratio": 0.0006313131307251751, |
| "epoch": 0.12097214636810485, |
| "grad_norm": 0.09582193195819855, |
| "kl": 0.033672553807264194, |
| "learning_rate": 6.000000000000001e-07, |
| "loss": 0.0001, |
| "step": 886 |
| }, |
| { |
| "clip_ratio": 0.0006313131307251751, |
| "epoch": 0.12110868377935555, |
| "grad_norm": 0.09397005289793015, |
| "kl": 0.03385620258632116, |
| "learning_rate": 5.947368421052632e-07, |
| "loss": 0.0002, |
| "step": 887 |
| }, |
| { |
| "clip_ratio": 0.00018825300503522158, |
| "epoch": 0.12124522119060623, |
| "grad_norm": 0.09514564275741577, |
| "kl": 0.03451714990660548, |
| "learning_rate": 5.894736842105263e-07, |
| "loss": 0.0001, |
| "step": 888 |
| }, |
| { |
| "clip_ratio": 0.00015862943837419152, |
| "completion_length": 124.9375, |
| "epoch": 0.1213817586018569, |
| "grad_norm": 0.08339914679527283, |
| "kl": 0.037555598362814635, |
| "learning_rate": 5.842105263157896e-07, |
| "loss": 0.0004, |
| "num_tokens": 2507443.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 889 |
| }, |
| { |
| "clip_ratio": 0.0005922052368987352, |
| "epoch": 0.1215182960131076, |
| "grad_norm": 0.08334753662347794, |
| "kl": 0.03709682886255905, |
| "learning_rate": 5.789473684210526e-07, |
| "loss": 0.0002, |
| "step": 890 |
| }, |
| { |
| "clip_ratio": 0.0005697215965483338, |
| "epoch": 0.12165483342435827, |
| "grad_norm": 0.08741384744644165, |
| "kl": 0.03697336750337854, |
| "learning_rate": 5.736842105263158e-07, |
| "loss": 0.0005, |
| "step": 891 |
| }, |
| { |
| "clip_ratio": 0.00018825300503522158, |
| "epoch": 0.12179137083560895, |
| "grad_norm": 0.08283984661102295, |
| "kl": 0.03640045766951516, |
| "learning_rate": 5.68421052631579e-07, |
| "loss": 0.0, |
| "step": 892 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 128.8125, |
| "epoch": 0.12192790824685965, |
| "grad_norm": 0.10741044580936432, |
| "kl": 0.03149555827258155, |
| "learning_rate": 5.631578947368421e-07, |
| "loss": 0.0003, |
| "num_tokens": 2518249.0, |
| "reward": 0.46562501788139343, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.46562501788139343, |
| "step": 893 |
| }, |
| { |
| "clip_ratio": 0.0006387395842466503, |
| "epoch": 0.12206444565811032, |
| "grad_norm": 0.10868413001298904, |
| "kl": 0.03139789224951528, |
| "learning_rate": 5.578947368421053e-07, |
| "loss": 0.0002, |
| "step": 894 |
| }, |
| { |
| "clip_ratio": 0.00042229730752296746, |
| "epoch": 0.122200983069361, |
| "grad_norm": 0.1074204370379448, |
| "kl": 0.03123481632792391, |
| "learning_rate": 5.526315789473684e-07, |
| "loss": 0.0001, |
| "step": 895 |
| }, |
| { |
| "clip_ratio": 0.00021114865376148373, |
| "epoch": 0.1223375204806117, |
| "grad_norm": 0.10637306421995163, |
| "kl": 0.03106152272084728, |
| "learning_rate": 5.473684210526316e-07, |
| "loss": 0.0, |
| "step": 896 |
| }, |
| { |
| "clip_ratio": 0.00020032051543239504, |
| "completion_length": 132.875, |
| "epoch": 0.12247405789186237, |
| "grad_norm": 0.06850402057170868, |
| "kl": 0.03606277168728411, |
| "learning_rate": 5.421052631578948e-07, |
| "loss": 0.0003, |
| "num_tokens": 2529705.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 897 |
| }, |
| { |
| "clip_ratio": 0.00022321428696159273, |
| "epoch": 0.12261059530311305, |
| "grad_norm": 0.06820071488618851, |
| "kl": 0.03548625635448843, |
| "learning_rate": 5.368421052631579e-07, |
| "loss": 0.0002, |
| "step": 898 |
| }, |
| { |
| "clip_ratio": 0.00020032051543239504, |
| "epoch": 0.12274713271436373, |
| "grad_norm": 0.0653720423579216, |
| "kl": 0.03564649049076252, |
| "learning_rate": 5.315789473684211e-07, |
| "loss": 0.0002, |
| "step": 899 |
| }, |
| { |
| "clip_ratio": 0.00020032051543239504, |
| "epoch": 0.12288367012561442, |
| "grad_norm": 0.0647495910525322, |
| "kl": 0.03579084068769589, |
| "learning_rate": 5.263157894736843e-07, |
| "loss": 0.0001, |
| "step": 900 |
| }, |
| { |
| "clip_ratio": 0.000712268622010015, |
| "completion_length": 141.59375, |
| "epoch": 0.1230202075368651, |
| "grad_norm": 0.13548840582370758, |
| "kl": 0.037526937725488096, |
| "learning_rate": 5.210526315789474e-07, |
| "loss": 0.0007, |
| "num_tokens": 2541880.0, |
| "reward": 0.49375003576278687, |
| "reward_std": 0.30735570192337036, |
| "rewards/reward_fn": 0.49375003576278687, |
| "step": 901 |
| }, |
| { |
| "clip_ratio": 0.001696178747806698, |
| "epoch": 0.12315674494811578, |
| "grad_norm": 0.13360342383384705, |
| "kl": 0.03651306492974982, |
| "learning_rate": 5.157894736842106e-07, |
| "loss": 0.0005, |
| "step": 902 |
| }, |
| { |
| "clip_ratio": 0.0004890543350484222, |
| "epoch": 0.12329328235936647, |
| "grad_norm": 0.13228605687618256, |
| "kl": 0.03699741544551216, |
| "learning_rate": 5.105263157894738e-07, |
| "loss": 0.0004, |
| "step": 903 |
| }, |
| { |
| "clip_ratio": 0.000613276133663021, |
| "epoch": 0.12342981977061715, |
| "grad_norm": 0.13343238830566406, |
| "kl": 0.03710320312529802, |
| "learning_rate": 5.052631578947369e-07, |
| "loss": 0.0004, |
| "step": 904 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 121.75, |
| "epoch": 0.12356635718186783, |
| "grad_norm": 0.10324323177337646, |
| "kl": 0.03289288666564971, |
| "learning_rate": 5.000000000000001e-07, |
| "loss": 0.0004, |
| "num_tokens": 2552476.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 905 |
| }, |
| { |
| "clip_ratio": 0.000510540179675445, |
| "epoch": 0.12370289459311852, |
| "grad_norm": 0.10418035089969635, |
| "kl": 0.03300356210093014, |
| "learning_rate": 4.947368421052632e-07, |
| "loss": 0.0003, |
| "step": 906 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "epoch": 0.1238394320043692, |
| "grad_norm": 0.10630179196596146, |
| "kl": 0.033201081794686615, |
| "learning_rate": 4.894736842105263e-07, |
| "loss": 0.0002, |
| "step": 907 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12397596941561988, |
| "grad_norm": 0.10277502238750458, |
| "kl": 0.032689066021703184, |
| "learning_rate": 4.842105263157895e-07, |
| "loss": 0.0, |
| "step": 908 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 118.21875, |
| "epoch": 0.12411250682687057, |
| "grad_norm": 0.06736943870782852, |
| "kl": 0.03596130615915172, |
| "learning_rate": 4.789473684210526e-07, |
| "loss": 0.0003, |
| "num_tokens": 2563159.0, |
| "reward": 0.9156249761581421, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.9156249761581421, |
| "step": 909 |
| }, |
| { |
| "clip_ratio": 0.00018825300503522158, |
| "epoch": 0.12424904423812125, |
| "grad_norm": 0.06819276511669159, |
| "kl": 0.03568592216470279, |
| "learning_rate": 4.7368421052631585e-07, |
| "loss": 0.0003, |
| "step": 910 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12438558164937193, |
| "grad_norm": 0.06739851832389832, |
| "kl": 0.035914964973926544, |
| "learning_rate": 4.6842105263157896e-07, |
| "loss": 0.0002, |
| "step": 911 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1245221190606226, |
| "grad_norm": 0.0662432387471199, |
| "kl": 0.03619830956449732, |
| "learning_rate": 4.631578947368422e-07, |
| "loss": 0.0002, |
| "step": 912 |
| }, |
| { |
| "clip_ratio": 0.0013353245303733274, |
| "completion_length": 120.15625, |
| "epoch": 0.1246586564718733, |
| "grad_norm": 0.12909458577632904, |
| "kl": 0.03360832063481212, |
| "learning_rate": 4.578947368421053e-07, |
| "loss": 0.0003, |
| "num_tokens": 2574240.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.3374999761581421, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 913 |
| }, |
| { |
| "clip_ratio": 0.0015245912363752723, |
| "epoch": 0.12479519388312398, |
| "grad_norm": 0.12952177226543427, |
| "kl": 0.03388803228153847, |
| "learning_rate": 4.526315789473685e-07, |
| "loss": 0.0002, |
| "step": 914 |
| }, |
| { |
| "clip_ratio": 0.0013912470312789083, |
| "epoch": 0.12493173129437465, |
| "grad_norm": 0.13111548125743866, |
| "kl": 0.03383615874918178, |
| "learning_rate": 4.473684210526316e-07, |
| "loss": 0.0, |
| "step": 915 |
| }, |
| { |
| "clip_ratio": 0.0012334425409790128, |
| "epoch": 0.12506826870562535, |
| "grad_norm": 0.1287565976381302, |
| "kl": 0.03395371185615659, |
| "learning_rate": 4.421052631578947e-07, |
| "loss": 0.0, |
| "step": 916 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 125.53125, |
| "epoch": 0.125204806116876, |
| "grad_norm": 0.10184290260076523, |
| "kl": 0.03474083298351616, |
| "learning_rate": 4.3684210526315794e-07, |
| "loss": 0.0001, |
| "num_tokens": 2585617.0, |
| "reward": 0.71875, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.71875, |
| "step": 917 |
| }, |
| { |
| "clip_ratio": 0.00023854961909819394, |
| "epoch": 0.1253413435281267, |
| "grad_norm": 0.10121199488639832, |
| "kl": 0.03448459054925479, |
| "learning_rate": 4.3157894736842105e-07, |
| "loss": 0.0002, |
| "step": 918 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1254778809393774, |
| "grad_norm": 0.09928610175848007, |
| "kl": 0.034620841237483546, |
| "learning_rate": 4.2631578947368427e-07, |
| "loss": 0.0, |
| "step": 919 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12561441835062806, |
| "grad_norm": 0.10057465732097626, |
| "kl": 0.03436506350408308, |
| "learning_rate": 4.210526315789474e-07, |
| "loss": 0.0, |
| "step": 920 |
| }, |
| { |
| "clip_ratio": 0.0002367424312978983, |
| "completion_length": 121.5, |
| "epoch": 0.12575095576187875, |
| "grad_norm": 0.03331659734249115, |
| "kl": 0.033621530630625784, |
| "learning_rate": 4.157894736842106e-07, |
| "loss": 0.0004, |
| "num_tokens": 2596589.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 921 |
| }, |
| { |
| "clip_ratio": 0.0002367424312978983, |
| "epoch": 0.12588749317312944, |
| "grad_norm": 0.03316027671098709, |
| "kl": 0.0338965390692465, |
| "learning_rate": 4.105263157894737e-07, |
| "loss": 0.0003, |
| "step": 922 |
| }, |
| { |
| "clip_ratio": 0.0002367424312978983, |
| "epoch": 0.1260240305843801, |
| "grad_norm": 0.033771876245737076, |
| "kl": 0.03396968066226691, |
| "learning_rate": 4.052631578947368e-07, |
| "loss": 0.0003, |
| "step": 923 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1261605679956308, |
| "grad_norm": 0.03320735692977905, |
| "kl": 0.03369491477496922, |
| "learning_rate": 4.0000000000000003e-07, |
| "loss": 0.0002, |
| "step": 924 |
| }, |
| { |
| "clip_ratio": 0.0008623184985481203, |
| "completion_length": 139.21875, |
| "epoch": 0.1262971054068815, |
| "grad_norm": 0.11683438718318939, |
| "kl": 0.04034199519082904, |
| "learning_rate": 3.9473684210526315e-07, |
| "loss": 0.0005, |
| "num_tokens": 2608672.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 925 |
| }, |
| { |
| "clip_ratio": 0.0004209743201499805, |
| "epoch": 0.12643364281813216, |
| "grad_norm": 0.13628651201725006, |
| "kl": 0.03980723247514106, |
| "learning_rate": 3.8947368421052636e-07, |
| "loss": 0.0005, |
| "step": 926 |
| }, |
| { |
| "clip_ratio": 0.0008091448398772627, |
| "epoch": 0.12657018022938285, |
| "grad_norm": 0.12596413493156433, |
| "kl": 0.03970417007803917, |
| "learning_rate": 3.8421052631578947e-07, |
| "loss": 0.0005, |
| "step": 927 |
| }, |
| { |
| "clip_ratio": 0.001051957588060759, |
| "epoch": 0.12670671764063354, |
| "grad_norm": 0.11056431382894516, |
| "kl": 0.039803695428417996, |
| "learning_rate": 3.789473684210527e-07, |
| "loss": 0.0005, |
| "step": 928 |
| }, |
| { |
| "clip_ratio": 0.0010233541834168136, |
| "completion_length": 140.59375, |
| "epoch": 0.1268432550518842, |
| "grad_norm": 0.08445189893245697, |
| "kl": 0.02938798555987887, |
| "learning_rate": 3.736842105263158e-07, |
| "loss": 0.0006, |
| "num_tokens": 2620411.0, |
| "reward": 0.578125, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.578125, |
| "step": 929 |
| }, |
| { |
| "clip_ratio": 0.00040687595901545137, |
| "epoch": 0.1269797924631349, |
| "grad_norm": 0.08192367851734161, |
| "kl": 0.029562115902081132, |
| "learning_rate": 3.6842105263157896e-07, |
| "loss": 0.0003, |
| "step": 930 |
| }, |
| { |
| "clip_ratio": 0.00026483050896786153, |
| "epoch": 0.1271163298743856, |
| "grad_norm": 0.08354966342449188, |
| "kl": 0.029616149666253477, |
| "learning_rate": 3.6315789473684213e-07, |
| "loss": 0.0004, |
| "step": 931 |
| }, |
| { |
| "clip_ratio": 0.0008278935856651515, |
| "epoch": 0.12725286728563626, |
| "grad_norm": 0.08334451168775558, |
| "kl": 0.029312406637473032, |
| "learning_rate": 3.578947368421053e-07, |
| "loss": 0.0004, |
| "step": 932 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 122.34375, |
| "epoch": 0.12738940469688695, |
| "grad_norm": 0.10578613728284836, |
| "kl": 0.028767388517735526, |
| "learning_rate": 3.5263157894736846e-07, |
| "loss": -0.0001, |
| "num_tokens": 2631774.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 933 |
| }, |
| { |
| "clip_ratio": 0.0002500000118743628, |
| "epoch": 0.12752594210813764, |
| "grad_norm": 0.10861129313707352, |
| "kl": 0.02848425370757468, |
| "learning_rate": 3.4736842105263157e-07, |
| "loss": 0.0001, |
| "step": 934 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1276624795193883, |
| "grad_norm": 0.09988290071487427, |
| "kl": 0.028927168430527672, |
| "learning_rate": 3.421052631578948e-07, |
| "loss": -0.0001, |
| "step": 935 |
| }, |
| { |
| "clip_ratio": 0.0002500000118743628, |
| "epoch": 0.127799016930639, |
| "grad_norm": 0.10976044088602066, |
| "kl": 0.029052481811959296, |
| "learning_rate": 3.368421052631579e-07, |
| "loss": -0.0003, |
| "step": 936 |
| }, |
| { |
| "clip_ratio": 0.00046764573198743165, |
| "completion_length": 149.4375, |
| "epoch": 0.1279355543418897, |
| "grad_norm": 0.1032148227095604, |
| "kl": 0.030085035221418366, |
| "learning_rate": 3.315789473684211e-07, |
| "loss": 0.0006, |
| "num_tokens": 2643816.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.35490381717681885, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 937 |
| }, |
| { |
| "clip_ratio": 0.0008250703540397808, |
| "epoch": 0.12807209175314035, |
| "grad_norm": 0.10333816707134247, |
| "kl": 0.029980021994560957, |
| "learning_rate": 3.263157894736842e-07, |
| "loss": 0.0004, |
| "step": 938 |
| }, |
| { |
| "clip_ratio": 0.0009608945401851088, |
| "epoch": 0.12820862916439105, |
| "grad_norm": 0.10449221730232239, |
| "kl": 0.030026076681679115, |
| "learning_rate": 3.210526315789474e-07, |
| "loss": 0.0004, |
| "step": 939 |
| }, |
| { |
| "clip_ratio": 0.0008269436075352132, |
| "epoch": 0.12834516657564174, |
| "grad_norm": 0.10373935848474503, |
| "kl": 0.02982323686592281, |
| "learning_rate": 3.1578947368421055e-07, |
| "loss": 0.0004, |
| "step": 940 |
| }, |
| { |
| "clip_ratio": 0.000616618781350553, |
| "completion_length": 114.5, |
| "epoch": 0.1284817039868924, |
| "grad_norm": 0.11087680608034134, |
| "kl": 0.0374185201653745, |
| "learning_rate": 3.105263157894737e-07, |
| "loss": 0.0002, |
| "num_tokens": 2654592.0, |
| "reward": 0.887499988079071, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.887499988079071, |
| "step": 941 |
| }, |
| { |
| "clip_ratio": 0.00026939655072055757, |
| "epoch": 0.1286182413981431, |
| "grad_norm": 0.12226645648479462, |
| "kl": 0.036657080316217616, |
| "learning_rate": 3.052631578947369e-07, |
| "loss": 0.0001, |
| "step": 942 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12875477880939376, |
| "grad_norm": 0.0905148908495903, |
| "kl": 0.03605576854897663, |
| "learning_rate": 3.0000000000000004e-07, |
| "loss": 0.0003, |
| "step": 943 |
| }, |
| { |
| "clip_ratio": 0.00026939655072055757, |
| "epoch": 0.12889131622064445, |
| "grad_norm": 0.09072388708591461, |
| "kl": 0.0357032545725815, |
| "learning_rate": 2.9473684210526315e-07, |
| "loss": 0.0001, |
| "step": 944 |
| }, |
| { |
| "clip_ratio": 0.0003004807804245502, |
| "completion_length": 125.3125, |
| "epoch": 0.12902785363189515, |
| "grad_norm": 0.1038588210940361, |
| "kl": 0.03439449705183506, |
| "learning_rate": 2.894736842105263e-07, |
| "loss": 0.0003, |
| "num_tokens": 2665466.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 945 |
| }, |
| { |
| "clip_ratio": 0.0007869734690757468, |
| "epoch": 0.1291643910431458, |
| "grad_norm": 0.10634484887123108, |
| "kl": 0.035002676682779565, |
| "learning_rate": 2.842105263157895e-07, |
| "loss": 0.0002, |
| "step": 946 |
| }, |
| { |
| "clip_ratio": 0.000922937979339622, |
| "epoch": 0.1293009284543965, |
| "grad_norm": 0.10326269268989563, |
| "kl": 0.03447505127405748, |
| "learning_rate": 2.7894736842105264e-07, |
| "loss": 0.0001, |
| "step": 947 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1294374658656472, |
| "grad_norm": 0.1021365150809288, |
| "kl": 0.03501308287377469, |
| "learning_rate": 2.736842105263158e-07, |
| "loss": -0.0001, |
| "step": 948 |
| }, |
| { |
| "clip_ratio": 0.00013469827536027879, |
| "completion_length": 138.09375, |
| "epoch": 0.12957400327689786, |
| "grad_norm": 0.07968829572200775, |
| "kl": 0.030057171738008037, |
| "learning_rate": 2.6842105263157897e-07, |
| "loss": 0.0003, |
| "num_tokens": 2677229.0, |
| "reward": 0.8875000476837158, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.8875000476837158, |
| "step": 949 |
| }, |
| { |
| "clip_ratio": 0.0006722042598994449, |
| "epoch": 0.12971054068814855, |
| "grad_norm": 0.0800618827342987, |
| "kl": 0.029572369530797005, |
| "learning_rate": 2.6315789473684213e-07, |
| "loss": 0.0004, |
| "step": 950 |
| }, |
| { |
| "clip_ratio": 0.0005265824729576707, |
| "epoch": 0.12984707809939924, |
| "grad_norm": 0.08067598938941956, |
| "kl": 0.029750974994385615, |
| "learning_rate": 2.578947368421053e-07, |
| "loss": 0.0002, |
| "step": 951 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1299836155106499, |
| "grad_norm": 0.07985977828502655, |
| "kl": 0.02976464355015196, |
| "learning_rate": 2.5263157894736846e-07, |
| "loss": 0.0002, |
| "step": 952 |
| }, |
| { |
| "clip_ratio": 0.0028174469189252704, |
| "completion_length": 130.96875, |
| "epoch": 0.1301201529219006, |
| "grad_norm": 2.5399577617645264, |
| "kl": 0.13722280028741807, |
| "learning_rate": 2.473684210526316e-07, |
| "loss": 0.0026, |
| "num_tokens": 2688792.0, |
| "reward": 0.546875, |
| "reward_std": 0.3111537992954254, |
| "rewards/reward_fn": 0.546875, |
| "step": 953 |
| }, |
| { |
| "clip_ratio": 0.0007530120201408863, |
| "epoch": 0.1302566903331513, |
| "grad_norm": 0.9123467206954956, |
| "kl": 0.0975925590028055, |
| "learning_rate": 2.4210526315789473e-07, |
| "loss": 0.0007, |
| "step": 954 |
| }, |
| { |
| "clip_ratio": 0.0003077960864175111, |
| "epoch": 0.13039322774440196, |
| "grad_norm": 0.7623748779296875, |
| "kl": 0.11674012421281077, |
| "learning_rate": 2.3684210526315792e-07, |
| "loss": 0.0004, |
| "step": 955 |
| }, |
| { |
| "clip_ratio": 0.0006403644947567955, |
| "epoch": 0.13052976515565265, |
| "grad_norm": 0.6193297505378723, |
| "kl": 0.09545972407795489, |
| "learning_rate": 2.315789473684211e-07, |
| "loss": 0.0002, |
| "step": 956 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.96875, |
| "epoch": 0.13066630256690334, |
| "grad_norm": 0.07178813219070435, |
| "kl": 0.03629979118704796, |
| "learning_rate": 2.2631578947368425e-07, |
| "loss": 0.0002, |
| "num_tokens": 2699387.0, |
| "reward": 0.9437500238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.9437500238418579, |
| "step": 957 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.130802839978154, |
| "grad_norm": 0.07103423029184341, |
| "kl": 0.03656398045131937, |
| "learning_rate": 2.2105263157894736e-07, |
| "loss": 0.0004, |
| "step": 958 |
| }, |
| { |
| "clip_ratio": 0.00025826445198617876, |
| "epoch": 0.1309393773894047, |
| "grad_norm": 0.0723857656121254, |
| "kl": 0.03695206393604167, |
| "learning_rate": 2.1578947368421053e-07, |
| "loss": 0.0002, |
| "step": 959 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1310759148006554, |
| "grad_norm": 0.07231997698545456, |
| "kl": 0.0363530462491326, |
| "learning_rate": 2.105263157894737e-07, |
| "loss": 0.0002, |
| "step": 960 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.84375, |
| "epoch": 0.13121245221190606, |
| "grad_norm": 0.09480443596839905, |
| "kl": 0.027393472104449756, |
| "learning_rate": 2.0526315789473685e-07, |
| "loss": 0.0002, |
| "num_tokens": 2710114.0, |
| "reward": 0.887499988079071, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.887499988079071, |
| "step": 961 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.13134898962315675, |
| "grad_norm": 0.09404343366622925, |
| "kl": 0.027128341323987115, |
| "learning_rate": 2.0000000000000002e-07, |
| "loss": 0.0001, |
| "step": 962 |
| }, |
| { |
| "clip_ratio": 0.0008934933575801551, |
| "epoch": 0.13148552703440744, |
| "grad_norm": 0.09405073523521423, |
| "kl": 0.027535340806934983, |
| "learning_rate": 1.9473684210526318e-07, |
| "loss": 0.0001, |
| "step": 963 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1316220644456581, |
| "grad_norm": 0.08958923071622849, |
| "kl": 0.027381854291888885, |
| "learning_rate": 1.8947368421052634e-07, |
| "loss": -0.0001, |
| "step": 964 |
| }, |
| { |
| "clip_ratio": 0.00042271205165889114, |
| "completion_length": 123.78125, |
| "epoch": 0.1317586018569088, |
| "grad_norm": 0.09731235355138779, |
| "kl": 0.02951424481580034, |
| "learning_rate": 1.8421052631578948e-07, |
| "loss": 0.0003, |
| "num_tokens": 2721579.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 965 |
| }, |
| { |
| "clip_ratio": 0.0006630534771829844, |
| "epoch": 0.1318951392681595, |
| "grad_norm": 0.0930672436952591, |
| "kl": 0.029122414154699072, |
| "learning_rate": 1.7894736842105265e-07, |
| "loss": 0.0003, |
| "step": 966 |
| }, |
| { |
| "clip_ratio": 0.0005357142654247582, |
| "epoch": 0.13203167667941015, |
| "grad_norm": 0.09273795038461685, |
| "kl": 0.02928502397844568, |
| "learning_rate": 1.7368421052631578e-07, |
| "loss": 0.0004, |
| "step": 967 |
| }, |
| { |
| "clip_ratio": 0.0007173645281000063, |
| "epoch": 0.13216821409066085, |
| "grad_norm": 0.09362951666116714, |
| "kl": 0.029012724291533232, |
| "learning_rate": 1.6842105263157895e-07, |
| "loss": 0.0003, |
| "step": 968 |
| }, |
| { |
| "clip_ratio": 0.0003330294566694647, |
| "completion_length": 126.40625, |
| "epoch": 0.1323047515019115, |
| "grad_norm": 0.08251232653856277, |
| "kl": 0.03195799436070956, |
| "learning_rate": 1.631578947368421e-07, |
| "loss": 0.0005, |
| "num_tokens": 2732936.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 969 |
| }, |
| { |
| "clip_ratio": 0.00035325744829606265, |
| "epoch": 0.1324412889131622, |
| "grad_norm": 0.08156436681747437, |
| "kl": 0.031294897315092385, |
| "learning_rate": 1.5789473684210527e-07, |
| "loss": 0.0002, |
| "step": 970 |
| }, |
| { |
| "clip_ratio": 0.0004885020898655057, |
| "epoch": 0.1325778263244129, |
| "grad_norm": 0.08326704800128937, |
| "kl": 0.03114733239635825, |
| "learning_rate": 1.5263157894736844e-07, |
| "loss": 0.0002, |
| "step": 971 |
| }, |
| { |
| "clip_ratio": 0.0011412299354560673, |
| "epoch": 0.13271436373566356, |
| "grad_norm": 0.08228829503059387, |
| "kl": 0.03161389287561178, |
| "learning_rate": 1.4736842105263158e-07, |
| "loss": 0.0003, |
| "step": 972 |
| }, |
| { |
| "clip_ratio": 0.0002741228090599179, |
| "completion_length": 118.3125, |
| "epoch": 0.13285090114691425, |
| "grad_norm": 0.11592817306518555, |
| "kl": 0.026773742283694446, |
| "learning_rate": 1.4210526315789474e-07, |
| "loss": 0.0001, |
| "num_tokens": 2743618.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 973 |
| }, |
| { |
| "clip_ratio": 0.0005201858002692461, |
| "epoch": 0.13298743855816494, |
| "grad_norm": 0.11682640016078949, |
| "kl": 0.027475335708004422, |
| "learning_rate": 1.368421052631579e-07, |
| "loss": 0.0002, |
| "step": 974 |
| }, |
| { |
| "clip_ratio": 0.0002741228090599179, |
| "epoch": 0.1331239759694156, |
| "grad_norm": 0.11784546077251434, |
| "kl": 0.027234470340772532, |
| "learning_rate": 1.3157894736842107e-07, |
| "loss": 0.0001, |
| "step": 975 |
| }, |
| { |
| "clip_ratio": 0.0002741228090599179, |
| "epoch": 0.1332605133806663, |
| "grad_norm": 0.10642171651124954, |
| "kl": 0.027347801194991916, |
| "learning_rate": 1.2631578947368423e-07, |
| "loss": -0.0001, |
| "step": 976 |
| }, |
| { |
| "clip_ratio": 0.0009640677599236369, |
| "completion_length": 122.6875, |
| "epoch": 0.133397050791917, |
| "grad_norm": 0.13537083566188812, |
| "kl": 0.02874184341635555, |
| "learning_rate": 1.2105263157894737e-07, |
| "loss": 0.0002, |
| "num_tokens": 2754564.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.4587019085884094, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 977 |
| }, |
| { |
| "clip_ratio": 0.0013094727764837444, |
| "epoch": 0.13353358820316766, |
| "grad_norm": 0.13505354523658752, |
| "kl": 0.028210823540575802, |
| "learning_rate": 1.1578947368421054e-07, |
| "loss": 0.0003, |
| "step": 978 |
| }, |
| { |
| "clip_ratio": 0.0004968734428985044, |
| "epoch": 0.13367012561441835, |
| "grad_norm": 0.13402731716632843, |
| "kl": 0.02882817276986316, |
| "learning_rate": 1.1052631578947368e-07, |
| "loss": 0.0001, |
| "step": 979 |
| }, |
| { |
| "clip_ratio": 0.0010141732927877456, |
| "epoch": 0.13380666302566904, |
| "grad_norm": 0.13402649760246277, |
| "kl": 0.027990734379272908, |
| "learning_rate": 1.0526315789473685e-07, |
| "loss": 0.0001, |
| "step": 980 |
| }, |
| { |
| "clip_ratio": 0.0004788851802004501, |
| "completion_length": 127.625, |
| "epoch": 0.1339432004369197, |
| "grad_norm": 0.08159787952899933, |
| "kl": 0.028808709408622235, |
| "learning_rate": 1.0000000000000001e-07, |
| "loss": -0.0, |
| "num_tokens": 2765624.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 981 |
| }, |
| { |
| "clip_ratio": 0.0002540650311857462, |
| "epoch": 0.1340797378481704, |
| "grad_norm": 0.08206771314144135, |
| "kl": 0.029088294133543968, |
| "learning_rate": 9.473684210526317e-08, |
| "loss": 0.0002, |
| "step": 982 |
| }, |
| { |
| "clip_ratio": 0.0002540650311857462, |
| "epoch": 0.1342162752594211, |
| "grad_norm": 0.08167807012796402, |
| "kl": 0.028653373738052323, |
| "learning_rate": 8.947368421052632e-08, |
| "loss": 0.0003, |
| "step": 983 |
| }, |
| { |
| "clip_ratio": 0.0004788851802004501, |
| "epoch": 0.13435281267067176, |
| "grad_norm": 0.08144602179527283, |
| "kl": 0.028931573644513264, |
| "learning_rate": 8.421052631578947e-08, |
| "loss": 0.0001, |
| "step": 984 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.25, |
| "epoch": 0.13448935008192245, |
| "grad_norm": 0.06735821813344955, |
| "kl": 0.029827186255715787, |
| "learning_rate": 7.894736842105264e-08, |
| "loss": 0.0004, |
| "num_tokens": 2775960.0, |
| "reward": 0.859375, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.859375, |
| "step": 985 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.13462588749317314, |
| "grad_norm": 0.06738390773534775, |
| "kl": 0.02973687121993862, |
| "learning_rate": 7.368421052631579e-08, |
| "loss": 0.0004, |
| "step": 986 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1347624249044238, |
| "grad_norm": 0.06684897094964981, |
| "kl": 0.029811298853019252, |
| "learning_rate": 6.842105263157895e-08, |
| "loss": 0.0003, |
| "step": 987 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1348989623156745, |
| "grad_norm": 0.06784624606370926, |
| "kl": 0.029368669900577515, |
| "learning_rate": 6.315789473684211e-08, |
| "loss": 0.0004, |
| "step": 988 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 129.90625, |
| "epoch": 0.1350354997269252, |
| "grad_norm": 0.08901333808898926, |
| "kl": 0.032253742014290765, |
| "learning_rate": 5.789473684210527e-08, |
| "loss": 0.0002, |
| "num_tokens": 2787289.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 989 |
| }, |
| { |
| "clip_ratio": 0.0004521122755249962, |
| "epoch": 0.13517203713817585, |
| "grad_norm": 0.08875369280576706, |
| "kl": 0.03244263178203255, |
| "learning_rate": 5.263157894736842e-08, |
| "loss": 0.0004, |
| "step": 990 |
| }, |
| { |
| "clip_ratio": 0.000534784237970598, |
| "epoch": 0.13530857454942655, |
| "grad_norm": 0.08818361163139343, |
| "kl": 0.032299379439791664, |
| "learning_rate": 4.7368421052631586e-08, |
| "loss": 0.0004, |
| "step": 991 |
| }, |
| { |
| "clip_ratio": 0.0009257595957024023, |
| "epoch": 0.13544511196067724, |
| "grad_norm": 0.0888131707906723, |
| "kl": 0.03237587565672584, |
| "learning_rate": 4.2105263157894737e-08, |
| "loss": 0.0002, |
| "step": 992 |
| }, |
| { |
| "clip_ratio": 0.000294811325147748, |
| "completion_length": 124.8125, |
| "epoch": 0.1355816493719279, |
| "grad_norm": 0.10416112095117569, |
| "kl": 0.029586980352178216, |
| "learning_rate": 3.6842105263157894e-08, |
| "loss": 0.0004, |
| "num_tokens": 2798919.0, |
| "reward": 0.71875, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.71875, |
| "step": 993 |
| }, |
| { |
| "clip_ratio": 0.0009029616194311529, |
| "epoch": 0.1357181867831786, |
| "grad_norm": 0.10409634560346603, |
| "kl": 0.029650241776835173, |
| "learning_rate": 3.157894736842106e-08, |
| "loss": 0.0004, |
| "step": 994 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.13585472419442926, |
| "grad_norm": 0.10347838699817657, |
| "kl": 0.02986885284190066, |
| "learning_rate": 2.631578947368421e-08, |
| "loss": 0.0003, |
| "step": 995 |
| }, |
| { |
| "clip_ratio": 0.00047837117745075375, |
| "epoch": 0.13599126160567995, |
| "grad_norm": 0.10247468948364258, |
| "kl": 0.029298015346284956, |
| "learning_rate": 2.1052631578947368e-08, |
| "loss": 0.0006, |
| "step": 996 |
| }, |
| { |
| "completion_length": 119.46875, |
| "epoch": 0.13612779901693065, |
| "grad_norm": 0.0, |
| "learning_rate": 1.578947368421053e-08, |
| "loss": 0.0, |
| "num_tokens": 2810074.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.0, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 997 |
| }, |
| { |
| "epoch": 0.1362643364281813, |
| "grad_norm": 0.0, |
| "learning_rate": 1.0526315789473684e-08, |
| "loss": 0.0, |
| "step": 998 |
| }, |
| { |
| "epoch": 0.136400873839432, |
| "grad_norm": 0.0, |
| "learning_rate": 5.263157894736842e-09, |
| "loss": 0.0, |
| "step": 999 |
| }, |
| { |
| "epoch": 0.1365374112506827, |
| "grad_norm": 0.0, |
| "learning_rate": 0.0, |
| "loss": 0.0, |
| "step": 1000 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 1000, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 250, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 32, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
