| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.1365374112506827, |
| "eval_steps": 500, |
| "global_step": 1000, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.0625, |
| "epoch": 0.00013653741125068268, |
| "grad_norm": 0.04404649883508682, |
| "kl": 0.0006317789470813295, |
| "learning_rate": 1.0000000000000001e-07, |
| "loss": -0.0002, |
| "num_tokens": 11046.0, |
| "reward": 0.7437499761581421, |
| "reward_std": 0.19240379333496094, |
| "rewards/reward_fn": 0.7437499761581421, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.00027307482250136535, |
| "grad_norm": 0.044033974409103394, |
| "kl": 0.0006317789470813295, |
| "learning_rate": 2.0000000000000002e-07, |
| "loss": -0.0002, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.0005441066459752619, |
| "epoch": 0.00040961223375204805, |
| "grad_norm": 0.04593058302998543, |
| "kl": 0.0006338120897453337, |
| "learning_rate": 3.0000000000000004e-07, |
| "loss": 0.0001, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 0.0004309742362238467, |
| "epoch": 0.0005461496450027307, |
| "grad_norm": 0.04400075227022171, |
| "kl": 0.0006654185117440647, |
| "learning_rate": 4.0000000000000003e-07, |
| "loss": -0.0, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 121.65625, |
| "epoch": 0.0006826870562534135, |
| "grad_norm": 0.04167269542813301, |
| "kl": 0.0005200268865337421, |
| "learning_rate": 5.000000000000001e-07, |
| "loss": -0.0002, |
| "num_tokens": 21963.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.0003748300950974226, |
| "epoch": 0.0008192244675040961, |
| "grad_norm": 0.041511062532663345, |
| "kl": 0.0005754963417530234, |
| "learning_rate": 6.000000000000001e-07, |
| "loss": -0.0001, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.00014334861771203578, |
| "epoch": 0.0009557618787547789, |
| "grad_norm": 0.04256992042064667, |
| "kl": 0.0005897999662920483, |
| "learning_rate": 7.000000000000001e-07, |
| "loss": -0.0002, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.00014334861771203578, |
| "epoch": 0.0010922992900054614, |
| "grad_norm": 0.04422915726900101, |
| "kl": 0.0005615864843093732, |
| "learning_rate": 8.000000000000001e-07, |
| "loss": -0.0001, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.0003235886397305876, |
| "completion_length": 150.625, |
| "epoch": 0.0012288367012561442, |
| "grad_norm": 0.05205063149333, |
| "kl": 0.0006086766702537716, |
| "learning_rate": 9.000000000000001e-07, |
| "loss": 0.0003, |
| "num_tokens": 34095.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.0006901410961290821, |
| "epoch": 0.001365374112506827, |
| "grad_norm": 0.0525423027575016, |
| "kl": 0.0005313775695867662, |
| "learning_rate": 1.0000000000000002e-06, |
| "loss": 0.0002, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.00037094928848091513, |
| "epoch": 0.0015019115237575095, |
| "grad_norm": 0.0524350069463253, |
| "kl": 0.0005592049219558248, |
| "learning_rate": 1.1e-06, |
| "loss": 0.0002, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.0006636033649556339, |
| "epoch": 0.0016384489350081922, |
| "grad_norm": 0.05291429162025452, |
| "kl": 0.0005318351923051523, |
| "learning_rate": 1.2000000000000002e-06, |
| "loss": 0.0002, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.000894870245247148, |
| "completion_length": 143.25, |
| "epoch": 0.001774986346258875, |
| "grad_norm": 0.042783912271261215, |
| "kl": 0.000602996098677977, |
| "learning_rate": 1.3e-06, |
| "loss": -0.0002, |
| "num_tokens": 45803.0, |
| "reward": 0.6031249761581421, |
| "reward_std": 0.2273801863193512, |
| "rewards/reward_fn": 0.6031249761581421, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.0005283939681248739, |
| "epoch": 0.0019115237575095577, |
| "grad_norm": 0.04314034804701805, |
| "kl": 0.0005507616770046297, |
| "learning_rate": 1.4000000000000001e-06, |
| "loss": -0.0001, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.00048371356388088316, |
| "epoch": 0.0020480611687602405, |
| "grad_norm": 0.043080903589725494, |
| "kl": 0.0005427919222711353, |
| "learning_rate": 1.5e-06, |
| "loss": -0.0001, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.0007240981794893742, |
| "epoch": 0.002184598580010923, |
| "grad_norm": 0.043031949549913406, |
| "kl": 0.0005427112396318989, |
| "learning_rate": 1.6000000000000001e-06, |
| "loss": -0.0002, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.0003906250058207661, |
| "completion_length": 126.28125, |
| "epoch": 0.0023211359912616056, |
| "grad_norm": 0.046606943011283875, |
| "kl": 0.0005118271424180421, |
| "learning_rate": 1.7000000000000002e-06, |
| "loss": 0.0001, |
| "num_tokens": 57356.0, |
| "reward": 0.6031249761581421, |
| "reward_std": 0.2273801863193512, |
| "rewards/reward_fn": 0.6031249761581421, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.0005387931014411151, |
| "epoch": 0.0024576734025122883, |
| "grad_norm": 0.04589732363820076, |
| "kl": 0.0005505777762664366, |
| "learning_rate": 1.8000000000000001e-06, |
| "loss": -0.0002, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.0013152419123798609, |
| "epoch": 0.002594210813762971, |
| "grad_norm": 0.04802809655666351, |
| "kl": 0.0005537249126064125, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": -0.0, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.00026939655072055757, |
| "epoch": 0.002730748225013654, |
| "grad_norm": 0.04628505930304527, |
| "kl": 0.0005950775475866976, |
| "learning_rate": 2.0000000000000003e-06, |
| "loss": -0.0001, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.0002816366613842547, |
| "completion_length": 169.25, |
| "epoch": 0.0028672856362643366, |
| "grad_norm": 0.02746995911002159, |
| "kl": 0.0005389471075432084, |
| "learning_rate": 2.1000000000000002e-06, |
| "loss": 0.0001, |
| "num_tokens": 69816.0, |
| "reward": 0.6031249761581421, |
| "reward_std": 0.1798321008682251, |
| "rewards/reward_fn": 0.6031249761581421, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.0002747003600234166, |
| "epoch": 0.003003823047515019, |
| "grad_norm": 0.027599724009633064, |
| "kl": 0.0005556085870921379, |
| "learning_rate": 2.2e-06, |
| "loss": 0.0001, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.0002166678459616378, |
| "epoch": 0.0031403604587657017, |
| "grad_norm": 0.027672842144966125, |
| "kl": 0.0006011502136971103, |
| "learning_rate": 2.3000000000000004e-06, |
| "loss": 0.0, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.0001949064462678507, |
| "epoch": 0.0032768978700163844, |
| "grad_norm": 0.027515968307852745, |
| "kl": 0.0005651746050716611, |
| "learning_rate": 2.4000000000000003e-06, |
| "loss": 0.0, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.0011138609115732834, |
| "completion_length": 132.90625, |
| "epoch": 0.003413435281267067, |
| "grad_norm": 0.061658527702093124, |
| "kl": 0.0005030216393606679, |
| "learning_rate": 2.5e-06, |
| "loss": -0.0, |
| "num_tokens": 81009.0, |
| "reward": 0.4906249940395355, |
| "reward_std": 0.361153781414032, |
| "rewards/reward_fn": 0.4906249940395355, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.00015394088404718786, |
| "epoch": 0.00354997269251775, |
| "grad_norm": 0.06129985302686691, |
| "kl": 0.0005159041643310047, |
| "learning_rate": 2.6e-06, |
| "loss": -0.0, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.0009837770485319197, |
| "epoch": 0.0036865101037684327, |
| "grad_norm": 0.06056167930364609, |
| "kl": 0.0005005182893000892, |
| "learning_rate": 2.7000000000000004e-06, |
| "loss": -0.0003, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.0003192847943864763, |
| "epoch": 0.0038230475150191155, |
| "grad_norm": 0.06112593412399292, |
| "kl": 0.0005432764692159253, |
| "learning_rate": 2.8000000000000003e-06, |
| "loss": 0.0001, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.0007011218112893403, |
| "completion_length": 136.0, |
| "epoch": 0.003959584926269798, |
| "grad_norm": 0.042925164103507996, |
| "kl": 0.0005347469618754985, |
| "learning_rate": 2.9e-06, |
| "loss": -0.0002, |
| "num_tokens": 92589.0, |
| "reward": 0.578125, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.578125, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.0005730880075134337, |
| "epoch": 0.004096122337520481, |
| "grad_norm": 0.04255054518580437, |
| "kl": 0.0004988939076611132, |
| "learning_rate": 3e-06, |
| "loss": -0.0002, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.00038095634954515845, |
| "epoch": 0.004232659748771164, |
| "grad_norm": 0.041760604828596115, |
| "kl": 0.0005955219580755511, |
| "learning_rate": 3.1000000000000004e-06, |
| "loss": -0.0001, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.00035894995380658656, |
| "epoch": 0.004369197160021846, |
| "grad_norm": 0.04276338592171669, |
| "kl": 0.0005454154365907016, |
| "learning_rate": 3.2000000000000003e-06, |
| "loss": -0.0001, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.0005223539337748662, |
| "completion_length": 139.84375, |
| "epoch": 0.004505734571272528, |
| "grad_norm": 0.04047078639268875, |
| "kl": 0.0005635294310195604, |
| "learning_rate": 3.3000000000000006e-06, |
| "loss": -0.0003, |
| "num_tokens": 104860.0, |
| "reward": 0.6000000238418579, |
| "reward_std": 0.1875, |
| "rewards/reward_fn": 0.6000000238418579, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.0005981834838166833, |
| "epoch": 0.004642271982523211, |
| "grad_norm": 0.03990301862359047, |
| "kl": 0.0006050642782611249, |
| "learning_rate": 3.4000000000000005e-06, |
| "loss": 0.0, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.0006553326529683545, |
| "epoch": 0.004778809393773894, |
| "grad_norm": 0.04005695506930351, |
| "kl": 0.0005889787116757361, |
| "learning_rate": 3.5e-06, |
| "loss": 0.0001, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.0010568553843768314, |
| "epoch": 0.004915346805024577, |
| "grad_norm": 0.03705165162682533, |
| "kl": 0.0006316659646472544, |
| "learning_rate": 3.6000000000000003e-06, |
| "loss": -0.0001, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.0008666114445077255, |
| "completion_length": 143.90625, |
| "epoch": 0.005051884216275259, |
| "grad_norm": 0.04572843387722969, |
| "kl": 0.0006531125750370848, |
| "learning_rate": 3.7e-06, |
| "loss": 0.0002, |
| "num_tokens": 116261.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.0005300959892338142, |
| "epoch": 0.005188421627525942, |
| "grad_norm": 0.0452241487801075, |
| "kl": 0.0006565239577867032, |
| "learning_rate": 3.8000000000000005e-06, |
| "loss": 0.0004, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.0022541209909832105, |
| "epoch": 0.005324959038776625, |
| "grad_norm": 0.04716404527425766, |
| "kl": 0.0006506023796646332, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": 0.0001, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.005461496450027308, |
| "grad_norm": 0.04658842459321022, |
| "kl": 0.0006605184144063969, |
| "learning_rate": 4.000000000000001e-06, |
| "loss": 0.0002, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.00045023551501799375, |
| "completion_length": 137.59375, |
| "epoch": 0.00559803386127799, |
| "grad_norm": 0.033117618411779404, |
| "kl": 0.0005335891364666168, |
| "learning_rate": 4.1e-06, |
| "loss": -0.0, |
| "num_tokens": 127732.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.005734571272528673, |
| "grad_norm": 0.032751549035310745, |
| "kl": 0.0005601238685812859, |
| "learning_rate": 4.2000000000000004e-06, |
| "loss": 0.0, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.0002962946309708059, |
| "epoch": 0.005871108683779356, |
| "grad_norm": 0.032139480113983154, |
| "kl": 0.0005848987057106569, |
| "learning_rate": 4.3e-06, |
| "loss": 0.0001, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.00015394088404718786, |
| "epoch": 0.006007646095030038, |
| "grad_norm": 0.033234309405088425, |
| "kl": 0.0005669799465977121, |
| "learning_rate": 4.4e-06, |
| "loss": -0.0, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 128.75, |
| "epoch": 0.006144183506280721, |
| "grad_norm": 0.03212456777691841, |
| "kl": 0.0005351848803911707, |
| "learning_rate": 4.5e-06, |
| "loss": 0.0001, |
| "num_tokens": 139392.0, |
| "reward": 0.4937499761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.4937499761581421, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.00021404109429568052, |
| "epoch": 0.006280720917531403, |
| "grad_norm": 0.032061729580163956, |
| "kl": 0.0005519898631973774, |
| "learning_rate": 4.600000000000001e-06, |
| "loss": 0.0001, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.006417258328782086, |
| "grad_norm": 0.032368190586566925, |
| "kl": 0.0006082248801249079, |
| "learning_rate": 4.7e-06, |
| "loss": -0.0001, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.006553795740032769, |
| "grad_norm": 0.032693859189748764, |
| "kl": 0.000633916805327317, |
| "learning_rate": 4.800000000000001e-06, |
| "loss": -0.0, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.0014423276152228937, |
| "completion_length": 131.03125, |
| "epoch": 0.006690333151283452, |
| "grad_norm": 0.0367547981441021, |
| "kl": 0.0006129781445451954, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": 0.0002, |
| "num_tokens": 150485.0, |
| "reward": 0.6343750357627869, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.6343750357627869, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.001040541916154325, |
| "epoch": 0.006826870562534134, |
| "grad_norm": 0.036152370274066925, |
| "kl": 0.0006829816738900263, |
| "learning_rate": 5e-06, |
| "loss": 0.0001, |
| "step": 50 |
| }, |
| { |
| "clip_ratio": 0.0013235063233878464, |
| "epoch": 0.006963407973784817, |
| "grad_norm": 0.03676420822739601, |
| "kl": 0.000648366946279566, |
| "learning_rate": 4.9947368421052636e-06, |
| "loss": 0.0001, |
| "step": 51 |
| }, |
| { |
| "clip_ratio": 0.0015096765419002622, |
| "epoch": 0.0070999453850355, |
| "grad_norm": 0.03675369918346405, |
| "kl": 0.0007533269481427851, |
| "learning_rate": 4.989473684210527e-06, |
| "loss": 0.0001, |
| "step": 52 |
| }, |
| { |
| "clip_ratio": 0.0013586657150881365, |
| "completion_length": 150.84375, |
| "epoch": 0.007236482796286183, |
| "grad_norm": 0.04119734838604927, |
| "kl": 0.000827303274490987, |
| "learning_rate": 4.98421052631579e-06, |
| "loss": 0.0, |
| "num_tokens": 162060.0, |
| "reward": 0.32499998807907104, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.32499998807907104, |
| "step": 53 |
| }, |
| { |
| "clip_ratio": 0.0007712610822636634, |
| "epoch": 0.007373020207536865, |
| "grad_norm": 0.040338173508644104, |
| "kl": 0.0008526074952897034, |
| "learning_rate": 4.978947368421053e-06, |
| "loss": -0.0001, |
| "step": 54 |
| }, |
| { |
| "clip_ratio": 0.0010123830288648605, |
| "epoch": 0.007509557618787548, |
| "grad_norm": 0.04008761793375015, |
| "kl": 0.0009329278500445071, |
| "learning_rate": 4.973684210526316e-06, |
| "loss": 0.0, |
| "step": 55 |
| }, |
| { |
| "clip_ratio": 0.0007876386807765812, |
| "epoch": 0.007646095030038231, |
| "grad_norm": 0.041053466498851776, |
| "kl": 0.0009653468450778746, |
| "learning_rate": 4.968421052631579e-06, |
| "loss": -0.0, |
| "step": 56 |
| }, |
| { |
| "clip_ratio": 0.00032552084303461015, |
| "completion_length": 124.5625, |
| "epoch": 0.007782632441288913, |
| "grad_norm": 0.026425430551171303, |
| "kl": 0.0010332740348530933, |
| "learning_rate": 4.9631578947368426e-06, |
| "loss": -0.0, |
| "num_tokens": 173450.0, |
| "reward": 0.578125, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.578125, |
| "step": 57 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.007919169852539596, |
| "grad_norm": 0.0263393335044384, |
| "kl": 0.0011578807752812281, |
| "learning_rate": 4.957894736842106e-06, |
| "loss": -0.0, |
| "step": 58 |
| }, |
| { |
| "clip_ratio": 0.00022163119865581393, |
| "epoch": 0.008055707263790278, |
| "grad_norm": 0.026425888761878014, |
| "kl": 0.001108602366912237, |
| "learning_rate": 4.952631578947369e-06, |
| "loss": 0.0, |
| "step": 59 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.008192244675040962, |
| "grad_norm": 0.026390153914690018, |
| "kl": 0.001177284767436504, |
| "learning_rate": 4.947368421052632e-06, |
| "loss": -0.0001, |
| "step": 60 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 131.4375, |
| "epoch": 0.008328782086291644, |
| "grad_norm": 0.023017792031168938, |
| "kl": 0.001508800451119896, |
| "learning_rate": 4.942105263157895e-06, |
| "loss": 0.0, |
| "num_tokens": 185108.0, |
| "reward": 0.971875011920929, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.971875011920929, |
| "step": 61 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.008465319497542327, |
| "grad_norm": 0.023074960336089134, |
| "kl": 0.0016179116428247653, |
| "learning_rate": 4.936842105263158e-06, |
| "loss": 0.0, |
| "step": 62 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.00860185690879301, |
| "grad_norm": 0.022814730182290077, |
| "kl": 0.0015702727341704303, |
| "learning_rate": 4.9315789473684215e-06, |
| "loss": -0.0001, |
| "step": 63 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.008738394320043691, |
| "grad_norm": 0.022533485665917397, |
| "kl": 0.001534115264803404, |
| "learning_rate": 4.926315789473685e-06, |
| "loss": -0.0001, |
| "step": 64 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.09375, |
| "epoch": 0.008874931731294375, |
| "grad_norm": 0.057550907135009766, |
| "kl": 0.0013586470904556336, |
| "learning_rate": 4.921052631578948e-06, |
| "loss": -0.0, |
| "num_tokens": 196231.0, |
| "reward": 0.578125, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.578125, |
| "step": 65 |
| }, |
| { |
| "clip_ratio": 0.00047872339200694114, |
| "epoch": 0.009011469142545057, |
| "grad_norm": 0.05568210408091545, |
| "kl": 0.0013682831449841615, |
| "learning_rate": 4.915789473684211e-06, |
| "loss": -0.0002, |
| "step": 66 |
| }, |
| { |
| "clip_ratio": 0.0007792041724314913, |
| "epoch": 0.00914800655379574, |
| "grad_norm": 0.0577705018222332, |
| "kl": 0.001314382108830614, |
| "learning_rate": 4.910526315789474e-06, |
| "loss": -0.0001, |
| "step": 67 |
| }, |
| { |
| "clip_ratio": 0.00047872339200694114, |
| "epoch": 0.009284543965046422, |
| "grad_norm": 0.05892620235681534, |
| "kl": 0.001385147581459023, |
| "learning_rate": 4.905263157894737e-06, |
| "loss": -0.0002, |
| "step": 68 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 132.3125, |
| "epoch": 0.009421081376297106, |
| "grad_norm": 0.05112706869840622, |
| "kl": 0.0014030390939296922, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": 0.0001, |
| "num_tokens": 207353.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 69 |
| }, |
| { |
| "clip_ratio": 0.00026483050896786153, |
| "epoch": 0.009557618787547788, |
| "grad_norm": 0.05096312239766121, |
| "kl": 0.0012969282797712367, |
| "learning_rate": 4.894736842105264e-06, |
| "loss": 0.0001, |
| "step": 70 |
| }, |
| { |
| "clip_ratio": 0.00018382353300694376, |
| "epoch": 0.009694156198798471, |
| "grad_norm": 0.050329335033893585, |
| "kl": 0.0012523090481408872, |
| "learning_rate": 4.889473684210527e-06, |
| "loss": -0.0, |
| "step": 71 |
| }, |
| { |
| "clip_ratio": 0.0005750250711571425, |
| "epoch": 0.009830693610049153, |
| "grad_norm": 0.05031115561723709, |
| "kl": 0.001234882418430061, |
| "learning_rate": 4.88421052631579e-06, |
| "loss": -0.0002, |
| "step": 72 |
| }, |
| { |
| "clip_ratio": 0.0015611131238983944, |
| "completion_length": 168.84375, |
| "epoch": 0.009967231021299837, |
| "grad_norm": 0.03763704001903534, |
| "kl": 0.0011852186817122856, |
| "learning_rate": 4.878947368421053e-06, |
| "loss": 0.0001, |
| "num_tokens": 220420.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 73 |
| }, |
| { |
| "clip_ratio": 0.0006092379480833188, |
| "epoch": 0.010103768432550519, |
| "grad_norm": 0.038565851747989655, |
| "kl": 0.001152727742010029, |
| "learning_rate": 4.873684210526316e-06, |
| "loss": 0.0001, |
| "step": 74 |
| }, |
| { |
| "clip_ratio": 0.0004241235728841275, |
| "epoch": 0.0102403058438012, |
| "grad_norm": 0.03837636113166809, |
| "kl": 0.0011011406968464144, |
| "learning_rate": 4.8684210526315795e-06, |
| "loss": -0.0001, |
| "step": 75 |
| }, |
| { |
| "clip_ratio": 0.0007302628655452281, |
| "epoch": 0.010376843255051884, |
| "grad_norm": 0.03833800554275513, |
| "kl": 0.0011365342761564534, |
| "learning_rate": 4.863157894736843e-06, |
| "loss": -0.0, |
| "step": 76 |
| }, |
| { |
| "clip_ratio": 0.0005017876101192087, |
| "completion_length": 167.21875, |
| "epoch": 0.010513380666302566, |
| "grad_norm": 0.035921789705753326, |
| "kl": 0.0010251052781313774, |
| "learning_rate": 4.857894736842106e-06, |
| "loss": -0.0001, |
| "num_tokens": 233075.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 77 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.01064991807755325, |
| "grad_norm": 0.0366053581237793, |
| "kl": 0.000940227837418206, |
| "learning_rate": 4.852631578947369e-06, |
| "loss": 0.0001, |
| "step": 78 |
| }, |
| { |
| "clip_ratio": 0.00014334861771203578, |
| "epoch": 0.010786455488803932, |
| "grad_norm": 0.03628067672252655, |
| "kl": 0.0008974663924163906, |
| "learning_rate": 4.847368421052631e-06, |
| "loss": 0.0001, |
| "step": 79 |
| }, |
| { |
| "clip_ratio": 0.00014334861771203578, |
| "epoch": 0.010922992900054615, |
| "grad_norm": 0.036221105605363846, |
| "kl": 0.0009643443936511176, |
| "learning_rate": 4.842105263157895e-06, |
| "loss": -0.0, |
| "step": 80 |
| }, |
| { |
| "clip_ratio": 0.0006383533182088286, |
| "completion_length": 142.46875, |
| "epoch": 0.011059530311305297, |
| "grad_norm": 0.03957719728350639, |
| "kl": 0.0008739960194361629, |
| "learning_rate": 4.8368421052631585e-06, |
| "loss": 0.0001, |
| "num_tokens": 244398.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 81 |
| }, |
| { |
| "clip_ratio": 0.00045246517402119935, |
| "epoch": 0.01119606772255598, |
| "grad_norm": 0.03962494060397148, |
| "kl": 0.0009068695708265295, |
| "learning_rate": 4.831578947368422e-06, |
| "loss": 0.0001, |
| "step": 82 |
| }, |
| { |
| "clip_ratio": 0.0003289473825134337, |
| "epoch": 0.011332605133806663, |
| "grad_norm": 0.0386803038418293, |
| "kl": 0.0009377624605804158, |
| "learning_rate": 4.826315789473685e-06, |
| "loss": 0.0, |
| "step": 83 |
| }, |
| { |
| "clip_ratio": 0.00016447369125671685, |
| "epoch": 0.011469142545057346, |
| "grad_norm": 0.039579425007104874, |
| "kl": 0.0010205690896327724, |
| "learning_rate": 4.821052631578948e-06, |
| "loss": 0.0001, |
| "step": 84 |
| }, |
| { |
| "clip_ratio": 0.0002765486715361476, |
| "completion_length": 159.625, |
| "epoch": 0.011605679956308028, |
| "grad_norm": 0.053214702755212784, |
| "kl": 0.0010023334107245319, |
| "learning_rate": 4.815789473684211e-06, |
| "loss": -0.0002, |
| "num_tokens": 256422.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.30735570192337036, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 85 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.011742217367558712, |
| "grad_norm": 0.052612077444791794, |
| "kl": 0.0010503011353648617, |
| "learning_rate": 4.8105263157894735e-06, |
| "loss": -0.0001, |
| "step": 86 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.011878754778809394, |
| "grad_norm": 0.052743155509233475, |
| "kl": 0.0009978616781154415, |
| "learning_rate": 4.8052631578947375e-06, |
| "loss": -0.0002, |
| "step": 87 |
| }, |
| { |
| "clip_ratio": 0.00034722223062999547, |
| "epoch": 0.012015292190060076, |
| "grad_norm": 0.05309215933084488, |
| "kl": 0.0010069226736959536, |
| "learning_rate": 4.800000000000001e-06, |
| "loss": -0.0003, |
| "step": 88 |
| }, |
| { |
| "clip_ratio": 0.00021853146608918905, |
| "completion_length": 142.6875, |
| "epoch": 0.01215182960131076, |
| "grad_norm": 0.05287536233663559, |
| "kl": 0.0009455939052713802, |
| "learning_rate": 4.794736842105264e-06, |
| "loss": 0.0003, |
| "num_tokens": 268864.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 89 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.012288367012561441, |
| "grad_norm": 0.04746906831860542, |
| "kl": 0.0009043413947438239, |
| "learning_rate": 4.789473684210527e-06, |
| "loss": 0.0002, |
| "step": 90 |
| }, |
| { |
| "clip_ratio": 0.0009161151392618194, |
| "epoch": 0.012424904423812125, |
| "grad_norm": 0.0458303801715374, |
| "kl": 0.0011064159052693867, |
| "learning_rate": 4.78421052631579e-06, |
| "loss": 0.0002, |
| "step": 91 |
| }, |
| { |
| "clip_ratio": 0.00042081948777195066, |
| "epoch": 0.012561441835062807, |
| "grad_norm": 0.05037999153137207, |
| "kl": 0.0009333853995485697, |
| "learning_rate": 4.778947368421053e-06, |
| "loss": 0.0, |
| "step": 92 |
| }, |
| { |
| "clip_ratio": 0.00033358884684275836, |
| "completion_length": 136.3125, |
| "epoch": 0.01269797924631349, |
| "grad_norm": 0.050271060317754745, |
| "kl": 0.001129487034631893, |
| "learning_rate": 4.773684210526316e-06, |
| "loss": 0.0001, |
| "num_tokens": 280298.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.578125, |
| "step": 93 |
| }, |
| { |
| "clip_ratio": 0.00017655367264524102, |
| "epoch": 0.012834516657564172, |
| "grad_norm": 0.050804417580366135, |
| "kl": 0.0011622968368101283, |
| "learning_rate": 4.76842105263158e-06, |
| "loss": -0.0, |
| "step": 94 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.012971054068814856, |
| "grad_norm": 0.050866566598415375, |
| "kl": 0.0010445236557643511, |
| "learning_rate": 4.763157894736842e-06, |
| "loss": -0.0001, |
| "step": 95 |
| }, |
| { |
| "clip_ratio": 0.00017655367264524102, |
| "epoch": 0.013107591480065538, |
| "grad_norm": 0.047069042921066284, |
| "kl": 0.0010408942343929084, |
| "learning_rate": 4.757894736842106e-06, |
| "loss": -0.0003, |
| "step": 96 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 153.3125, |
| "epoch": 0.013244128891316221, |
| "grad_norm": 0.04759960621595383, |
| "kl": 0.0009134336842180346, |
| "learning_rate": 4.752631578947369e-06, |
| "loss": -0.0002, |
| "num_tokens": 292492.0, |
| "reward": 0.71875, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.71875, |
| "step": 97 |
| }, |
| { |
| "clip_ratio": 0.0003427685151109472, |
| "epoch": 0.013380666302566903, |
| "grad_norm": 0.047368988394737244, |
| "kl": 0.0008575451474825968, |
| "learning_rate": 4.747368421052632e-06, |
| "loss": -0.0, |
| "step": 98 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.013517203713817587, |
| "grad_norm": 0.04754631221294403, |
| "kl": 0.0009104160817514639, |
| "learning_rate": 4.7421052631578954e-06, |
| "loss": -0.0001, |
| "step": 99 |
| }, |
| { |
| "clip_ratio": 0.0003645833348855376, |
| "epoch": 0.013653741125068269, |
| "grad_norm": 0.047607775777578354, |
| "kl": 0.0009035689445227035, |
| "learning_rate": 4.736842105263158e-06, |
| "loss": -0.0002, |
| "step": 100 |
| }, |
| { |
| "clip_ratio": 0.0007289039349416271, |
| "completion_length": 165.59375, |
| "epoch": 0.01379027853631895, |
| "grad_norm": 0.03178010880947113, |
| "kl": 0.000884010269146529, |
| "learning_rate": 4.731578947368422e-06, |
| "loss": 0.0002, |
| "num_tokens": 305279.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 101 |
| }, |
| { |
| "clip_ratio": 0.0007626213628100231, |
| "epoch": 0.013926815947569634, |
| "grad_norm": 0.03182770684361458, |
| "kl": 0.0009488556288488326, |
| "learning_rate": 4.726315789473684e-06, |
| "loss": -0.0, |
| "step": 102 |
| }, |
| { |
| "clip_ratio": 0.00015624999650754035, |
| "epoch": 0.014063353358820316, |
| "grad_norm": 0.03163674473762512, |
| "kl": 0.0009546505116304616, |
| "learning_rate": 4.721052631578948e-06, |
| "loss": 0.0, |
| "step": 103 |
| }, |
| { |
| "clip_ratio": 0.00015624999650754035, |
| "epoch": 0.014199890770071, |
| "grad_norm": 0.031845323741436005, |
| "kl": 0.0009727316901262384, |
| "learning_rate": 4.71578947368421e-06, |
| "loss": -0.0, |
| "step": 104 |
| }, |
| { |
| "clip_ratio": 0.0007236909732455388, |
| "completion_length": 158.53125, |
| "epoch": 0.014336428181321682, |
| "grad_norm": 0.0269012413918972, |
| "kl": 0.0010989235524903052, |
| "learning_rate": 4.710526315789474e-06, |
| "loss": 0.0002, |
| "num_tokens": 317712.0, |
| "reward": 0.9156249761581421, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.9156249761581421, |
| "step": 105 |
| }, |
| { |
| "clip_ratio": 0.0006901180895511061, |
| "epoch": 0.014472965592572365, |
| "grad_norm": 0.027830632403492928, |
| "kl": 0.0011699929145834176, |
| "learning_rate": 4.705263157894738e-06, |
| "loss": 0.0003, |
| "step": 106 |
| }, |
| { |
| "clip_ratio": 0.00045146002958063036, |
| "epoch": 0.014609503003823047, |
| "grad_norm": 0.027583377435803413, |
| "kl": 0.001156308660938521, |
| "learning_rate": 4.7e-06, |
| "loss": 0.0002, |
| "step": 107 |
| }, |
| { |
| "clip_ratio": 0.00028153153834864497, |
| "epoch": 0.01474604041507373, |
| "grad_norm": 0.02736779861152172, |
| "kl": 0.0011434779335104395, |
| "learning_rate": 4.694736842105264e-06, |
| "loss": 0.0002, |
| "step": 108 |
| }, |
| { |
| "clip_ratio": 0.00020695364219136536, |
| "completion_length": 152.46875, |
| "epoch": 0.014882577826324413, |
| "grad_norm": 0.037834905087947845, |
| "kl": 0.0011261462568654679, |
| "learning_rate": 4.689473684210526e-06, |
| "loss": 0.0001, |
| "num_tokens": 329931.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 109 |
| }, |
| { |
| "clip_ratio": 0.0004254851082805544, |
| "epoch": 0.015019115237575096, |
| "grad_norm": 0.03780336678028107, |
| "kl": 0.0010323524820705643, |
| "learning_rate": 4.68421052631579e-06, |
| "loss": 0.0001, |
| "step": 110 |
| }, |
| { |
| "clip_ratio": 0.0004254851082805544, |
| "epoch": 0.015155652648825778, |
| "grad_norm": 0.03730948641896248, |
| "kl": 0.0011414301479817368, |
| "learning_rate": 4.6789473684210525e-06, |
| "loss": -0.0, |
| "step": 111 |
| }, |
| { |
| "clip_ratio": 0.000155472633196041, |
| "epoch": 0.015292190060076462, |
| "grad_norm": 0.03791780024766922, |
| "kl": 0.001147796273471613, |
| "learning_rate": 4.6736842105263166e-06, |
| "loss": -0.0, |
| "step": 112 |
| }, |
| { |
| "clip_ratio": 0.00037912980769760907, |
| "completion_length": 155.78125, |
| "epoch": 0.015428727471327144, |
| "grad_norm": 0.038378458470106125, |
| "kl": 0.0012724736388918245, |
| "learning_rate": 4.668421052631579e-06, |
| "loss": 0.0, |
| "num_tokens": 342056.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 113 |
| }, |
| { |
| "clip_ratio": 0.0008178490679711103, |
| "epoch": 0.015565264882577826, |
| "grad_norm": 0.038206491619348526, |
| "kl": 0.001326220324699534, |
| "learning_rate": 4.663157894736842e-06, |
| "loss": -0.0001, |
| "step": 114 |
| }, |
| { |
| "clip_ratio": 0.000673972099320963, |
| "epoch": 0.015701802293828507, |
| "grad_norm": 0.03806019201874733, |
| "kl": 0.001337001162028173, |
| "learning_rate": 4.657894736842106e-06, |
| "loss": -0.0, |
| "step": 115 |
| }, |
| { |
| "clip_ratio": 0.0006484463519882411, |
| "epoch": 0.015838339705079193, |
| "grad_norm": 0.03847945109009743, |
| "kl": 0.0013689603329112288, |
| "learning_rate": 4.652631578947368e-06, |
| "loss": -0.0, |
| "step": 116 |
| }, |
| { |
| "clip_ratio": 0.0003124999930150807, |
| "completion_length": 140.03125, |
| "epoch": 0.015974877116329875, |
| "grad_norm": 0.053888190537691116, |
| "kl": 0.0015626058402631315, |
| "learning_rate": 4.647368421052632e-06, |
| "loss": -0.0, |
| "num_tokens": 353577.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 117 |
| }, |
| { |
| "clip_ratio": 0.0006850460486020893, |
| "epoch": 0.016111414527580557, |
| "grad_norm": 0.051420435309410095, |
| "kl": 0.0014924607939974521, |
| "learning_rate": 4.642105263157895e-06, |
| "loss": -0.0001, |
| "step": 118 |
| }, |
| { |
| "clip_ratio": 0.0006249999860301614, |
| "epoch": 0.01624795193883124, |
| "grad_norm": 0.05272446572780609, |
| "kl": 0.0014821061740803998, |
| "learning_rate": 4.636842105263159e-06, |
| "loss": -0.0003, |
| "step": 119 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.016384489350081924, |
| "grad_norm": 0.051526475697755814, |
| "kl": 0.001648830244448618, |
| "learning_rate": 4.631578947368421e-06, |
| "loss": -0.0004, |
| "step": 120 |
| }, |
| { |
| "clip_ratio": 0.00046813665539957583, |
| "completion_length": 157.6875, |
| "epoch": 0.016521026761332606, |
| "grad_norm": 0.06405887007713318, |
| "kl": 0.0015702300315751927, |
| "learning_rate": 4.626315789473684e-06, |
| "loss": -0.0002, |
| "num_tokens": 365611.0, |
| "reward": 0.5750000476837158, |
| "reward_std": 0.17499999701976776, |
| "rewards/reward_fn": 0.5750000476837158, |
| "step": 121 |
| }, |
| { |
| "clip_ratio": 0.0007894021546235308, |
| "epoch": 0.016657564172583288, |
| "grad_norm": 0.06610463559627533, |
| "kl": 0.0014424660766962916, |
| "learning_rate": 4.621052631578948e-06, |
| "loss": -0.0001, |
| "step": 122 |
| }, |
| { |
| "clip_ratio": 0.0006425309693440795, |
| "epoch": 0.01679410158383397, |
| "grad_norm": 0.04856649786233902, |
| "kl": 0.001490594368078746, |
| "learning_rate": 4.6157894736842105e-06, |
| "loss": -0.0003, |
| "step": 123 |
| }, |
| { |
| "clip_ratio": 0.00046395954268518835, |
| "epoch": 0.016930638995084655, |
| "grad_norm": 0.052688200026750565, |
| "kl": 0.0015647518521291204, |
| "learning_rate": 4.6105263157894745e-06, |
| "loss": -0.0004, |
| "step": 124 |
| }, |
| { |
| "clip_ratio": 0.00021701389050576836, |
| "completion_length": 139.71875, |
| "epoch": 0.017067176406335337, |
| "grad_norm": 0.03784525766968727, |
| "kl": 0.0014868502239551162, |
| "learning_rate": 4.605263157894737e-06, |
| "loss": -0.0, |
| "num_tokens": 377486.0, |
| "reward": 0.4375, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.4375, |
| "step": 125 |
| }, |
| { |
| "clip_ratio": 0.00026939655072055757, |
| "epoch": 0.01720371381758602, |
| "grad_norm": 0.039032455533742905, |
| "kl": 0.0016484676798427245, |
| "learning_rate": 4.600000000000001e-06, |
| "loss": -0.0001, |
| "step": 126 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0173402512288367, |
| "grad_norm": 0.038484662771224976, |
| "kl": 0.001675849733146606, |
| "learning_rate": 4.594736842105263e-06, |
| "loss": -0.0002, |
| "step": 127 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.017476788640087382, |
| "grad_norm": 0.03741098567843437, |
| "kl": 0.0017028618640324567, |
| "learning_rate": 4.589473684210526e-06, |
| "loss": -0.0003, |
| "step": 128 |
| }, |
| { |
| "completion_length": 135.15625, |
| "epoch": 0.017613326051338068, |
| "grad_norm": 0.0, |
| "learning_rate": 4.5842105263157895e-06, |
| "loss": 0.0, |
| "num_tokens": 389019.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.0, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 129 |
| }, |
| { |
| "epoch": 0.01774986346258875, |
| "grad_norm": 0.0, |
| "learning_rate": 4.578947368421053e-06, |
| "loss": 0.0, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.01788640087383943, |
| "grad_norm": 0.0, |
| "learning_rate": 4.573684210526317e-06, |
| "loss": 0.0, |
| "step": 131 |
| }, |
| { |
| "epoch": 0.018022938285090113, |
| "grad_norm": 0.0, |
| "learning_rate": 4.568421052631579e-06, |
| "loss": 0.0, |
| "step": 132 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 131.53125, |
| "epoch": 0.0181594756963408, |
| "grad_norm": 0.038026828318834305, |
| "kl": 0.0016733734537410783, |
| "learning_rate": 4.563157894736843e-06, |
| "loss": -0.0001, |
| "num_tokens": 400084.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 133 |
| }, |
| { |
| "clip_ratio": 0.0006313131307251751, |
| "epoch": 0.01829601310759148, |
| "grad_norm": 0.03821878135204315, |
| "kl": 0.0016837922430568142, |
| "learning_rate": 4.557894736842105e-06, |
| "loss": -0.0001, |
| "step": 134 |
| }, |
| { |
| "clip_ratio": 0.00031565656536258757, |
| "epoch": 0.018432550518842163, |
| "grad_norm": 0.035876594483852386, |
| "kl": 0.0017077570118999574, |
| "learning_rate": 4.552631578947369e-06, |
| "loss": -0.0002, |
| "step": 135 |
| }, |
| { |
| "clip_ratio": 0.00031565656536258757, |
| "epoch": 0.018569087930092845, |
| "grad_norm": 0.03872598335146904, |
| "kl": 0.0016680303797329543, |
| "learning_rate": 4.547368421052632e-06, |
| "loss": -0.0, |
| "step": 136 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 142.15625, |
| "epoch": 0.01870562534134353, |
| "grad_norm": 0.028256189078092575, |
| "kl": 0.0017378787051711697, |
| "learning_rate": 4.542105263157895e-06, |
| "loss": 0.0, |
| "num_tokens": 412021.0, |
| "reward": 0.9156249761581421, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.9156249761581421, |
| "step": 137 |
| }, |
| { |
| "clip_ratio": 0.00028641909011639655, |
| "epoch": 0.018842162752594212, |
| "grad_norm": 0.028107410296797752, |
| "kl": 0.001677423257206101, |
| "learning_rate": 4.536842105263158e-06, |
| "loss": 0.0, |
| "step": 138 |
| }, |
| { |
| "clip_ratio": 0.0002705300285015255, |
| "epoch": 0.018978700163844894, |
| "grad_norm": 0.027991972863674164, |
| "kl": 0.0016291097908833763, |
| "learning_rate": 4.531578947368421e-06, |
| "loss": -0.0, |
| "step": 139 |
| }, |
| { |
| "clip_ratio": 0.00012450199574232101, |
| "epoch": 0.019115237575095576, |
| "grad_norm": 0.02784087136387825, |
| "kl": 0.0016967972096608719, |
| "learning_rate": 4.526315789473685e-06, |
| "loss": -0.0, |
| "step": 140 |
| }, |
| { |
| "clip_ratio": 0.00015394088404718786, |
| "completion_length": 157.0625, |
| "epoch": 0.019251774986346257, |
| "grad_norm": 0.04372892528772354, |
| "kl": 0.0020676578678830992, |
| "learning_rate": 4.5210526315789475e-06, |
| "loss": 0.0002, |
| "num_tokens": 424671.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.578125, |
| "step": 141 |
| }, |
| { |
| "clip_ratio": 0.0003565062361303717, |
| "epoch": 0.019388312397596943, |
| "grad_norm": 0.04700653254985809, |
| "kl": 0.002142951811038074, |
| "learning_rate": 4.5157894736842115e-06, |
| "loss": 0.0, |
| "step": 142 |
| }, |
| { |
| "clip_ratio": 0.0004839111497858539, |
| "epoch": 0.019524849808847625, |
| "grad_norm": 0.04318815842270851, |
| "kl": 0.0021835622974322177, |
| "learning_rate": 4.510526315789474e-06, |
| "loss": -0.0001, |
| "step": 143 |
| }, |
| { |
| "clip_ratio": 0.00018939393339678645, |
| "epoch": 0.019661387220098307, |
| "grad_norm": 0.04352035000920296, |
| "kl": 0.0021402186721388716, |
| "learning_rate": 4.505263157894737e-06, |
| "loss": 0.0, |
| "step": 144 |
| }, |
| { |
| "clip_ratio": 0.0012175573210697621, |
| "completion_length": 123.0625, |
| "epoch": 0.01979792463134899, |
| "grad_norm": 0.04905979335308075, |
| "kl": 0.0017827195497375214, |
| "learning_rate": 4.5e-06, |
| "loss": 0.0002, |
| "num_tokens": 435205.0, |
| "reward": 0.71875, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.71875, |
| "step": 145 |
| }, |
| { |
| "clip_ratio": 0.0013242935965536162, |
| "epoch": 0.019934462042599674, |
| "grad_norm": 0.04945666342973709, |
| "kl": 0.0018286748254467966, |
| "learning_rate": 4.494736842105263e-06, |
| "loss": 0.0002, |
| "step": 146 |
| }, |
| { |
| "clip_ratio": 0.0005930003535468131, |
| "epoch": 0.020070999453850356, |
| "grad_norm": 0.049488019198179245, |
| "kl": 0.0018381225654593436, |
| "learning_rate": 4.489473684210527e-06, |
| "loss": 0.0001, |
| "step": 147 |
| }, |
| { |
| "clip_ratio": 0.0008262093033408746, |
| "epoch": 0.020207536865101038, |
| "grad_norm": 0.048830606043338776, |
| "kl": 0.0018431637563480763, |
| "learning_rate": 4.48421052631579e-06, |
| "loss": -0.0003, |
| "step": 148 |
| }, |
| { |
| "clip_ratio": 0.0009542925690766424, |
| "completion_length": 128.09375, |
| "epoch": 0.02034407427635172, |
| "grad_norm": 0.039013754576444626, |
| "kl": 0.0018213689982076176, |
| "learning_rate": 4.478947368421054e-06, |
| "loss": 0.0002, |
| "num_tokens": 446040.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 149 |
| }, |
| { |
| "clip_ratio": 0.00048096767568495125, |
| "epoch": 0.0204806116876024, |
| "grad_norm": 0.039544761180877686, |
| "kl": 0.0017687516847217921, |
| "learning_rate": 4.473684210526316e-06, |
| "loss": -0.0001, |
| "step": 150 |
| }, |
| { |
| "clip_ratio": 0.00014810427092015743, |
| "epoch": 0.020617149098853087, |
| "grad_norm": 0.038592442870140076, |
| "kl": 0.0018994503989233635, |
| "learning_rate": 4.468421052631579e-06, |
| "loss": -0.0002, |
| "step": 151 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.02075368651010377, |
| "grad_norm": 0.03908444941043854, |
| "kl": 0.0017993021683651023, |
| "learning_rate": 4.463157894736842e-06, |
| "loss": -0.0001, |
| "step": 152 |
| }, |
| { |
| "clip_ratio": 0.0005235375429037958, |
| "completion_length": 117.09375, |
| "epoch": 0.02089022392135445, |
| "grad_norm": 0.03812478110194206, |
| "kl": 0.0022767211703467183, |
| "learning_rate": 4.4578947368421054e-06, |
| "loss": -0.0002, |
| "num_tokens": 456967.0, |
| "reward": 0.4375000298023224, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.4375000298023224, |
| "step": 153 |
| }, |
| { |
| "clip_ratio": 0.0006113408599048853, |
| "epoch": 0.021026761332605132, |
| "grad_norm": 0.03862268850207329, |
| "kl": 0.0021923944495938485, |
| "learning_rate": 4.452631578947369e-06, |
| "loss": -0.0001, |
| "step": 154 |
| }, |
| { |
| "clip_ratio": 0.0006774784269509837, |
| "epoch": 0.021163298743855818, |
| "grad_norm": 0.03888405114412308, |
| "kl": 0.0022211558498383965, |
| "learning_rate": 4.447368421052632e-06, |
| "loss": -0.0001, |
| "step": 155 |
| }, |
| { |
| "clip_ratio": 0.000485546508571133, |
| "epoch": 0.0212998361551065, |
| "grad_norm": 0.037996355444192886, |
| "kl": 0.002334445103770122, |
| "learning_rate": 4.442105263157896e-06, |
| "loss": -0.0004, |
| "step": 156 |
| }, |
| { |
| "clip_ratio": 0.001523168000858277, |
| "completion_length": 133.8125, |
| "epoch": 0.02143637356635718, |
| "grad_norm": 0.056003473699092865, |
| "kl": 0.001776036844603368, |
| "learning_rate": 4.436842105263158e-06, |
| "loss": 0.0003, |
| "num_tokens": 467973.0, |
| "reward": 0.71875, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.71875, |
| "step": 157 |
| }, |
| { |
| "clip_ratio": 0.0005791426374344155, |
| "epoch": 0.021572910977607863, |
| "grad_norm": 0.055576346814632416, |
| "kl": 0.0018412170556985075, |
| "learning_rate": 4.431578947368421e-06, |
| "loss": 0.0001, |
| "step": 158 |
| }, |
| { |
| "clip_ratio": 0.0015547180810244754, |
| "epoch": 0.02170944838885855, |
| "grad_norm": 0.05685070902109146, |
| "kl": 0.0018155872894567437, |
| "learning_rate": 4.426315789473684e-06, |
| "loss": 0.0, |
| "step": 159 |
| }, |
| { |
| "clip_ratio": 0.0003484987828414887, |
| "epoch": 0.02184598580010923, |
| "grad_norm": 0.05709301307797432, |
| "kl": 0.0018661026006157044, |
| "learning_rate": 4.4210526315789476e-06, |
| "loss": -0.0004, |
| "step": 160 |
| }, |
| { |
| "clip_ratio": 0.00022163119865581393, |
| "completion_length": 137.0625, |
| "epoch": 0.021982523211359913, |
| "grad_norm": 0.05108444765210152, |
| "kl": 0.0018476451787137194, |
| "learning_rate": 4.415789473684211e-06, |
| "loss": 0.0002, |
| "num_tokens": 479703.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 161 |
| }, |
| { |
| "clip_ratio": 0.0005414267943706363, |
| "epoch": 0.022119060622610594, |
| "grad_norm": 0.050916336476802826, |
| "kl": 0.0019034737615584163, |
| "learning_rate": 4.410526315789474e-06, |
| "loss": 0.0, |
| "step": 162 |
| }, |
| { |
| "clip_ratio": 0.000384391620173119, |
| "epoch": 0.022255598033861276, |
| "grad_norm": 0.04771227389574051, |
| "kl": 0.0018498922854632838, |
| "learning_rate": 4.405263157894737e-06, |
| "loss": 0.0001, |
| "step": 163 |
| }, |
| { |
| "clip_ratio": 0.00028669723542407155, |
| "epoch": 0.02239213544511196, |
| "grad_norm": 0.04805860295891762, |
| "kl": 0.0019589400762924924, |
| "learning_rate": 4.4e-06, |
| "loss": -0.0001, |
| "step": 164 |
| }, |
| { |
| "clip_ratio": 0.0004507835255935788, |
| "completion_length": 140.3125, |
| "epoch": 0.022528672856362644, |
| "grad_norm": 0.0500267893075943, |
| "kl": 0.0017338303587166592, |
| "learning_rate": 4.394736842105263e-06, |
| "loss": -0.0, |
| "num_tokens": 491673.0, |
| "reward": 0.859375, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.859375, |
| "step": 165 |
| }, |
| { |
| "clip_ratio": 0.00021404109429568052, |
| "epoch": 0.022665210267613325, |
| "grad_norm": 0.04949698969721794, |
| "kl": 0.001850471655416186, |
| "learning_rate": 4.3894736842105266e-06, |
| "loss": -0.0001, |
| "step": 166 |
| }, |
| { |
| "clip_ratio": 0.000565689813811332, |
| "epoch": 0.022801747678864007, |
| "grad_norm": 0.048048555850982666, |
| "kl": 0.0017547677452967037, |
| "learning_rate": 4.38421052631579e-06, |
| "loss": -0.0001, |
| "step": 167 |
| }, |
| { |
| "clip_ratio": 0.0003289473825134337, |
| "epoch": 0.022938285090114693, |
| "grad_norm": 0.04920797049999237, |
| "kl": 0.0016859639854374109, |
| "learning_rate": 4.378947368421053e-06, |
| "loss": -0.0002, |
| "step": 168 |
| }, |
| { |
| "clip_ratio": 0.0008012236576178111, |
| "completion_length": 169.375, |
| "epoch": 0.023074822501365375, |
| "grad_norm": 0.05868339538574219, |
| "kl": 0.002092949611324002, |
| "learning_rate": 4.373684210526316e-06, |
| "loss": 0.0002, |
| "num_tokens": 504033.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.40245187282562256, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 169 |
| }, |
| { |
| "clip_ratio": 0.000662178557831794, |
| "epoch": 0.023211359912616056, |
| "grad_norm": 0.05874933674931526, |
| "kl": 0.002041788393398747, |
| "learning_rate": 4.368421052631579e-06, |
| "loss": -0.0, |
| "step": 170 |
| }, |
| { |
| "clip_ratio": 0.0007715479878243059, |
| "epoch": 0.02334789732386674, |
| "grad_norm": 0.05879794806241989, |
| "kl": 0.00202036062728439, |
| "learning_rate": 4.363157894736842e-06, |
| "loss": -0.0001, |
| "step": 171 |
| }, |
| { |
| "clip_ratio": 0.0005064057695562951, |
| "epoch": 0.023484434735117424, |
| "grad_norm": 0.05847965553402901, |
| "kl": 0.0021620094048557803, |
| "learning_rate": 4.3578947368421055e-06, |
| "loss": -0.0004, |
| "step": 172 |
| }, |
| { |
| "clip_ratio": 0.0004963119863532484, |
| "completion_length": 155.21875, |
| "epoch": 0.023620972146368106, |
| "grad_norm": 0.04289793223142624, |
| "kl": 0.002111369007252506, |
| "learning_rate": 4.352631578947369e-06, |
| "loss": 0.0003, |
| "num_tokens": 515752.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 173 |
| }, |
| { |
| "clip_ratio": 0.00023148147738538682, |
| "epoch": 0.023757509557618788, |
| "grad_norm": 0.04338355362415314, |
| "kl": 0.002090127652991214, |
| "learning_rate": 4.347368421052632e-06, |
| "loss": 0.0002, |
| "step": 174 |
| }, |
| { |
| "clip_ratio": 0.00023148147738538682, |
| "epoch": 0.02389404696886947, |
| "grad_norm": 0.043108005076646805, |
| "kl": 0.0020603093526005978, |
| "learning_rate": 4.342105263157895e-06, |
| "loss": 0.0001, |
| "step": 175 |
| }, |
| { |
| "clip_ratio": 0.00023148147738538682, |
| "epoch": 0.02403058438012015, |
| "grad_norm": 0.04241011291742325, |
| "kl": 0.0022579077194677666, |
| "learning_rate": 4.336842105263158e-06, |
| "loss": 0.0002, |
| "step": 176 |
| }, |
| { |
| "clip_ratio": 0.00010212418419541791, |
| "completion_length": 155.5625, |
| "epoch": 0.024167121791370837, |
| "grad_norm": 0.05044643208384514, |
| "kl": 0.00230243670011987, |
| "learning_rate": 4.331578947368421e-06, |
| "loss": 0.0, |
| "num_tokens": 527834.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 177 |
| }, |
| { |
| "clip_ratio": 0.0014079755346756428, |
| "epoch": 0.02430365920262152, |
| "grad_norm": 0.051211778074502945, |
| "kl": 0.0025526770295982715, |
| "learning_rate": 4.3263157894736845e-06, |
| "loss": 0.0001, |
| "step": 178 |
| }, |
| { |
| "clip_ratio": 0.0006058427607058547, |
| "epoch": 0.0244401966138722, |
| "grad_norm": 0.05046148598194122, |
| "kl": 0.0024460002277919557, |
| "learning_rate": 4.321052631578948e-06, |
| "loss": 0.0, |
| "step": 179 |
| }, |
| { |
| "clip_ratio": 0.00017755682347342372, |
| "epoch": 0.024576734025122882, |
| "grad_norm": 0.05063207820057869, |
| "kl": 0.0025408670771867037, |
| "learning_rate": 4.315789473684211e-06, |
| "loss": -0.0003, |
| "step": 180 |
| }, |
| { |
| "clip_ratio": 0.0005968966870568693, |
| "completion_length": 139.09375, |
| "epoch": 0.024713271436373568, |
| "grad_norm": 0.057065319269895554, |
| "kl": 0.002252304788271431, |
| "learning_rate": 4.310526315789474e-06, |
| "loss": 0.0001, |
| "num_tokens": 539861.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 181 |
| }, |
| { |
| "clip_ratio": 0.0005400872905738652, |
| "epoch": 0.02484980884762425, |
| "grad_norm": 0.05701858550310135, |
| "kl": 0.0022254607938521076, |
| "learning_rate": 4.305263157894737e-06, |
| "loss": 0.0, |
| "step": 182 |
| }, |
| { |
| "clip_ratio": 0.0003518342855386436, |
| "epoch": 0.02498634625887493, |
| "grad_norm": 0.05623921379446983, |
| "kl": 0.002288297408085782, |
| "learning_rate": 4.3e-06, |
| "loss": -0.0002, |
| "step": 183 |
| }, |
| { |
| "clip_ratio": 0.0008291525591630489, |
| "epoch": 0.025122883670125613, |
| "grad_norm": 0.05564498156309128, |
| "kl": 0.0023957003249961417, |
| "learning_rate": 4.2947368421052635e-06, |
| "loss": -0.0001, |
| "step": 184 |
| }, |
| { |
| "clip_ratio": 0.0002297794126207009, |
| "completion_length": 145.25, |
| "epoch": 0.0252594210813763, |
| "grad_norm": 0.047647591680288315, |
| "kl": 0.0022527406908920966, |
| "learning_rate": 4.289473684210527e-06, |
| "loss": 0.0003, |
| "num_tokens": 551857.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 185 |
| }, |
| { |
| "clip_ratio": 0.00017458100046496838, |
| "epoch": 0.02539595849262698, |
| "grad_norm": 0.04730217531323433, |
| "kl": 0.0022600671800319105, |
| "learning_rate": 4.28421052631579e-06, |
| "loss": 0.0001, |
| "step": 186 |
| }, |
| { |
| "clip_ratio": 0.000659806071780622, |
| "epoch": 0.025532495903877662, |
| "grad_norm": 0.04729049652814865, |
| "kl": 0.0023344330584222917, |
| "learning_rate": 4.278947368421053e-06, |
| "loss": 0.0, |
| "step": 187 |
| }, |
| { |
| "clip_ratio": 0.00017458100046496838, |
| "epoch": 0.025669033315128344, |
| "grad_norm": 0.04799634963274002, |
| "kl": 0.002339323709747987, |
| "learning_rate": 4.273684210526316e-06, |
| "loss": -0.0003, |
| "step": 188 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 142.28125, |
| "epoch": 0.025805570726379026, |
| "grad_norm": 0.032953836023807526, |
| "kl": 0.0026757955238281284, |
| "learning_rate": 4.268421052631579e-06, |
| "loss": 0.0, |
| "num_tokens": 562966.0, |
| "reward": 0.831250011920929, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.831250011920929, |
| "step": 189 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.02594210813762971, |
| "grad_norm": 0.0329064279794693, |
| "kl": 0.00255184544948861, |
| "learning_rate": 4.2631578947368425e-06, |
| "loss": -0.0001, |
| "step": 190 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.026078645548880394, |
| "grad_norm": 0.03271796181797981, |
| "kl": 0.0026399646885693073, |
| "learning_rate": 4.257894736842106e-06, |
| "loss": -0.0002, |
| "step": 191 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.026215182960131075, |
| "grad_norm": 0.03261572867631912, |
| "kl": 0.0026238768514303956, |
| "learning_rate": 4.252631578947369e-06, |
| "loss": -0.0001, |
| "step": 192 |
| }, |
| { |
| "clip_ratio": 0.0005445075803436339, |
| "completion_length": 135.1875, |
| "epoch": 0.026351720371381757, |
| "grad_norm": 0.04699130356311798, |
| "kl": 0.0023030067604850046, |
| "learning_rate": 4.247368421052632e-06, |
| "loss": -0.0, |
| "num_tokens": 573928.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 193 |
| }, |
| { |
| "clip_ratio": 0.00103052583290264, |
| "epoch": 0.026488257782632443, |
| "grad_norm": 0.047233857214450836, |
| "kl": 0.00226948533054383, |
| "learning_rate": 4.242105263157895e-06, |
| "loss": 0.0001, |
| "step": 194 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "epoch": 0.026624795193883125, |
| "grad_norm": 0.04670139402151108, |
| "kl": 0.0023675450538576115, |
| "learning_rate": 4.236842105263158e-06, |
| "loss": 0.0002, |
| "step": 195 |
| }, |
| { |
| "clip_ratio": 0.0011015485797543079, |
| "epoch": 0.026761332605133806, |
| "grad_norm": 0.046695202589035034, |
| "kl": 0.0022257951659412356, |
| "learning_rate": 4.2315789473684215e-06, |
| "loss": 0.0, |
| "step": 196 |
| }, |
| { |
| "clip_ratio": 0.00012400794366840273, |
| "completion_length": 138.46875, |
| "epoch": 0.02689787001638449, |
| "grad_norm": 0.03901498392224312, |
| "kl": 0.0027911317265534308, |
| "learning_rate": 4.226315789473685e-06, |
| "loss": -0.0001, |
| "num_tokens": 585215.0, |
| "reward": 0.6593749523162842, |
| "reward_std": 0.18370190262794495, |
| "rewards/reward_fn": 0.6593749523162842, |
| "step": 197 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.027034407427635174, |
| "grad_norm": 0.03807137534022331, |
| "kl": 0.0028723968789563514, |
| "learning_rate": 4.221052631578948e-06, |
| "loss": -0.0001, |
| "step": 198 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.027170944838885856, |
| "grad_norm": 0.03774174302816391, |
| "kl": 0.0028623725866054883, |
| "learning_rate": 4.215789473684211e-06, |
| "loss": -0.0002, |
| "step": 199 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.027307482250136537, |
| "grad_norm": 0.03775256127119064, |
| "kl": 0.0027698824978870107, |
| "learning_rate": 4.210526315789474e-06, |
| "loss": -0.0004, |
| "step": 200 |
| }, |
| { |
| "clip_ratio": 0.00013130252773407847, |
| "completion_length": 165.375, |
| "epoch": 0.02744401966138722, |
| "grad_norm": 0.019780190661549568, |
| "kl": 0.002381948237598408, |
| "learning_rate": 4.205263157894737e-06, |
| "loss": 0.0001, |
| "num_tokens": 597811.0, |
| "reward": 0.71875, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.71875, |
| "step": 201 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0275805570726379, |
| "grad_norm": 0.019986584782600403, |
| "kl": 0.0023435521106875967, |
| "learning_rate": 4.2000000000000004e-06, |
| "loss": -0.0, |
| "step": 202 |
| }, |
| { |
| "clip_ratio": 0.00026260505546815693, |
| "epoch": 0.027717094483888587, |
| "grad_norm": 0.020404886454343796, |
| "kl": 0.0023913175446068635, |
| "learning_rate": 4.194736842105264e-06, |
| "loss": 0.0, |
| "step": 203 |
| }, |
| { |
| "clip_ratio": 0.00019778481510002166, |
| "epoch": 0.02785363189513927, |
| "grad_norm": 0.020100388675928116, |
| "kl": 0.0024267902790597873, |
| "learning_rate": 4.189473684210527e-06, |
| "loss": 0.0, |
| "step": 204 |
| }, |
| { |
| "clip_ratio": 0.000506756769027561, |
| "completion_length": 158.0625, |
| "epoch": 0.02799016930638995, |
| "grad_norm": 0.041435953229665756, |
| "kl": 0.0023757141243549995, |
| "learning_rate": 4.18421052631579e-06, |
| "loss": -0.0001, |
| "num_tokens": 609853.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 205 |
| }, |
| { |
| "clip_ratio": 0.0004917787300655618, |
| "epoch": 0.028126706717640632, |
| "grad_norm": 0.04093041643500328, |
| "kl": 0.002551862231484847, |
| "learning_rate": 4.178947368421053e-06, |
| "loss": -0.0002, |
| "step": 206 |
| }, |
| { |
| "clip_ratio": 0.0001594387722434476, |
| "epoch": 0.028263244128891318, |
| "grad_norm": 0.04131149873137474, |
| "kl": 0.002450428895826917, |
| "learning_rate": 4.173684210526316e-06, |
| "loss": -0.0002, |
| "step": 207 |
| }, |
| { |
| "clip_ratio": 0.00015394088404718786, |
| "epoch": 0.028399781540142, |
| "grad_norm": 0.040568362921476364, |
| "kl": 0.0024964501499198377, |
| "learning_rate": 4.1684210526315794e-06, |
| "loss": -0.0003, |
| "step": 208 |
| }, |
| { |
| "clip_ratio": 0.0009547190275043249, |
| "completion_length": 146.8125, |
| "epoch": 0.02853631895139268, |
| "grad_norm": 0.04699120298027992, |
| "kl": 0.002375929558183998, |
| "learning_rate": 4.163157894736843e-06, |
| "loss": 0.0, |
| "num_tokens": 622387.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 209 |
| }, |
| { |
| "clip_ratio": 0.0006435829127440229, |
| "epoch": 0.028672856362643363, |
| "grad_norm": 0.04699277505278587, |
| "kl": 0.002440200205455767, |
| "learning_rate": 4.157894736842106e-06, |
| "loss": -0.0001, |
| "step": 210 |
| }, |
| { |
| "clip_ratio": 0.00011081559932790697, |
| "epoch": 0.02880939377389405, |
| "grad_norm": 0.04718649759888649, |
| "kl": 0.002467095066094771, |
| "learning_rate": 4.152631578947369e-06, |
| "loss": -0.0003, |
| "step": 211 |
| }, |
| { |
| "clip_ratio": 0.0003324467979837209, |
| "epoch": 0.02894593118514473, |
| "grad_norm": 0.046153001487255096, |
| "kl": 0.0024199721519835293, |
| "learning_rate": 4.147368421052632e-06, |
| "loss": -0.0005, |
| "step": 212 |
| }, |
| { |
| "clip_ratio": 0.00013586955901701003, |
| "completion_length": 162.1875, |
| "epoch": 0.029082468596395412, |
| "grad_norm": 0.03387923911213875, |
| "kl": 0.002338993905141251, |
| "learning_rate": 4.142105263157895e-06, |
| "loss": -0.0001, |
| "num_tokens": 635097.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 213 |
| }, |
| { |
| "clip_ratio": 0.00014204545004758984, |
| "epoch": 0.029219006007646094, |
| "grad_norm": 0.033278003334999084, |
| "kl": 0.0024614290523459204, |
| "learning_rate": 4.136842105263158e-06, |
| "loss": -0.0002, |
| "step": 214 |
| }, |
| { |
| "clip_ratio": 0.00040687595901545137, |
| "epoch": 0.029355543418896776, |
| "grad_norm": 0.03360047563910484, |
| "kl": 0.0025313771366199944, |
| "learning_rate": 4.1315789473684216e-06, |
| "loss": -0.0002, |
| "step": 215 |
| }, |
| { |
| "clip_ratio": 0.0005427455180324614, |
| "epoch": 0.02949208083014746, |
| "grad_norm": 0.03357577696442604, |
| "kl": 0.002459269828250399, |
| "learning_rate": 4.126315789473685e-06, |
| "loss": -0.0002, |
| "step": 216 |
| }, |
| { |
| "clip_ratio": 0.0008163582533597946, |
| "completion_length": 150.5, |
| "epoch": 0.029628618241398143, |
| "grad_norm": 0.05886125937104225, |
| "kl": 0.0026728738484962378, |
| "learning_rate": 4.121052631578948e-06, |
| "loss": -0.0001, |
| "num_tokens": 647273.0, |
| "reward": 0.71875, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.71875, |
| "step": 217 |
| }, |
| { |
| "clip_ratio": 0.00028669723542407155, |
| "epoch": 0.029765155652648825, |
| "grad_norm": 0.05972710996866226, |
| "kl": 0.0025688415862532565, |
| "learning_rate": 4.115789473684211e-06, |
| "loss": -0.0001, |
| "step": 218 |
| }, |
| { |
| "clip_ratio": 0.00021258502965793014, |
| "epoch": 0.029901693063899507, |
| "grad_norm": 0.0589703805744648, |
| "kl": 0.0027555712149478495, |
| "learning_rate": 4.110526315789474e-06, |
| "loss": -0.0003, |
| "step": 219 |
| }, |
| { |
| "clip_ratio": 0.00047741553862579167, |
| "epoch": 0.030038230475150193, |
| "grad_norm": 0.05902567878365517, |
| "kl": 0.00267484737560153, |
| "learning_rate": 4.105263157894737e-06, |
| "loss": -0.0003, |
| "step": 220 |
| }, |
| { |
| "clip_ratio": 0.0005867703148396686, |
| "completion_length": 154.28125, |
| "epoch": 0.030174767886400874, |
| "grad_norm": 0.06789493560791016, |
| "kl": 0.002492289610017906, |
| "learning_rate": 4.1e-06, |
| "loss": 0.0001, |
| "num_tokens": 659510.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 221 |
| }, |
| { |
| "clip_ratio": 0.001256241142982617, |
| "epoch": 0.030311305297651556, |
| "grad_norm": 0.03951876610517502, |
| "kl": 0.0021740910688095028, |
| "learning_rate": 4.094736842105264e-06, |
| "loss": 0.0001, |
| "step": 222 |
| }, |
| { |
| "clip_ratio": 0.0008764668309595436, |
| "epoch": 0.030447842708902238, |
| "grad_norm": 0.038591429591178894, |
| "kl": 0.002234173503893544, |
| "learning_rate": 4.089473684210527e-06, |
| "loss": 0.0001, |
| "step": 223 |
| }, |
| { |
| "clip_ratio": 0.000996052665868774, |
| "epoch": 0.030584380120152924, |
| "grad_norm": 0.038493406027555466, |
| "kl": 0.0023900925771158654, |
| "learning_rate": 4.08421052631579e-06, |
| "loss": -0.0001, |
| "step": 224 |
| }, |
| { |
| "clip_ratio": 0.0004790522070834413, |
| "completion_length": 172.0625, |
| "epoch": 0.030720917531403606, |
| "grad_norm": 0.04708952084183693, |
| "kl": 0.0027864959483849816, |
| "learning_rate": 4.078947368421053e-06, |
| "loss": 0.0001, |
| "num_tokens": 671788.0, |
| "reward": 0.71875, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.71875, |
| "step": 225 |
| }, |
| { |
| "clip_ratio": 0.0005403020913945511, |
| "epoch": 0.030857454942654287, |
| "grad_norm": 0.047040365636348724, |
| "kl": 0.002808534052746836, |
| "learning_rate": 4.073684210526316e-06, |
| "loss": -0.0, |
| "step": 226 |
| }, |
| { |
| "clip_ratio": 0.0004790522070834413, |
| "epoch": 0.03099399235390497, |
| "grad_norm": 0.04768039658665657, |
| "kl": 0.002784293676086236, |
| "learning_rate": 4.0684210526315795e-06, |
| "loss": -0.0003, |
| "step": 227 |
| }, |
| { |
| "clip_ratio": 0.00043576216557994485, |
| "epoch": 0.03113052976515565, |
| "grad_norm": 0.047202423214912415, |
| "kl": 0.002762944764981512, |
| "learning_rate": 4.063157894736842e-06, |
| "loss": -0.0002, |
| "step": 228 |
| }, |
| { |
| "clip_ratio": 0.0002338850244996138, |
| "completion_length": 145.6875, |
| "epoch": 0.03126706717640634, |
| "grad_norm": 0.052416592836380005, |
| "kl": 0.0029842189615010284, |
| "learning_rate": 4.057894736842106e-06, |
| "loss": 0.0001, |
| "num_tokens": 684122.0, |
| "reward": 0.4937500059604645, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.4937500059604645, |
| "step": 229 |
| }, |
| { |
| "clip_ratio": 0.0005076464294688776, |
| "epoch": 0.031403604587657015, |
| "grad_norm": 0.05282365158200264, |
| "kl": 0.0029398120859696064, |
| "learning_rate": 4.052631578947368e-06, |
| "loss": 0.0001, |
| "step": 230 |
| }, |
| { |
| "clip_ratio": 0.0009736515348777175, |
| "epoch": 0.0315401419989077, |
| "grad_norm": 0.04937496408820152, |
| "kl": 0.0029731830472883303, |
| "learning_rate": 4.047368421052632e-06, |
| "loss": -0.0003, |
| "step": 231 |
| }, |
| { |
| "clip_ratio": 0.00032849946001078933, |
| "epoch": 0.031676679410158386, |
| "grad_norm": 0.051063161343336105, |
| "kl": 0.002983107568070409, |
| "learning_rate": 4.042105263157895e-06, |
| "loss": -0.0005, |
| "step": 232 |
| }, |
| { |
| "clip_ratio": 0.00013766519259661436, |
| "completion_length": 193.59375, |
| "epoch": 0.031813216821409064, |
| "grad_norm": 0.03487762063741684, |
| "kl": 0.0024473068915540352, |
| "learning_rate": 4.0368421052631585e-06, |
| "loss": 0.0, |
| "num_tokens": 697541.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 233 |
| }, |
| { |
| "clip_ratio": 0.00013766519259661436, |
| "epoch": 0.03194975423265975, |
| "grad_norm": 0.03475506976246834, |
| "kl": 0.002526979300455423, |
| "learning_rate": 4.031578947368422e-06, |
| "loss": 0.0, |
| "step": 234 |
| }, |
| { |
| "clip_ratio": 0.00013766519259661436, |
| "epoch": 0.032086291643910435, |
| "grad_norm": 0.034642960876226425, |
| "kl": 0.0026074592606164515, |
| "learning_rate": 4.026315789473684e-06, |
| "loss": -0.0001, |
| "step": 235 |
| }, |
| { |
| "clip_ratio": 0.00024047692568274215, |
| "epoch": 0.03222282905516111, |
| "grad_norm": 0.03465104475617409, |
| "kl": 0.002620333540107822, |
| "learning_rate": 4.021052631578948e-06, |
| "loss": -0.0002, |
| "step": 236 |
| }, |
| { |
| "clip_ratio": 0.0003746633155969903, |
| "completion_length": 158.65625, |
| "epoch": 0.0323593664664118, |
| "grad_norm": 0.04539260268211365, |
| "kl": 0.00334534621288185, |
| "learning_rate": 4.01578947368421e-06, |
| "loss": 0.0002, |
| "num_tokens": 709754.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 237 |
| }, |
| { |
| "clip_ratio": 0.0008486020669806749, |
| "epoch": 0.03249590387766248, |
| "grad_norm": 0.04548077657818794, |
| "kl": 0.0035286891943542287, |
| "learning_rate": 4.010526315789474e-06, |
| "loss": -0.0001, |
| "step": 238 |
| }, |
| { |
| "clip_ratio": 0.00040586062823422253, |
| "epoch": 0.03263244128891316, |
| "grad_norm": 0.04545941948890686, |
| "kl": 0.0035132169650751166, |
| "learning_rate": 4.005263157894737e-06, |
| "loss": -0.0003, |
| "step": 239 |
| }, |
| { |
| "clip_ratio": 0.00044274143874645233, |
| "epoch": 0.03276897870016385, |
| "grad_norm": 0.04565747082233429, |
| "kl": 0.0037534972543653566, |
| "learning_rate": 4.000000000000001e-06, |
| "loss": -0.0001, |
| "step": 240 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 142.1875, |
| "epoch": 0.032905516111414526, |
| "grad_norm": 0.02257436327636242, |
| "kl": 0.003795149543293519, |
| "learning_rate": 3.994736842105264e-06, |
| "loss": -0.0, |
| "num_tokens": 721092.0, |
| "reward": 0.4937499761581421, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.4937499761581421, |
| "step": 241 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03304205352266521, |
| "grad_norm": 0.02336946688592434, |
| "kl": 0.0038620703453489114, |
| "learning_rate": 3.989473684210526e-06, |
| "loss": -0.0001, |
| "step": 242 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03317859093391589, |
| "grad_norm": 0.02494773082435131, |
| "kl": 0.003890741354553029, |
| "learning_rate": 3.98421052631579e-06, |
| "loss": -0.0001, |
| "step": 243 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.033315128345166575, |
| "grad_norm": 0.024358734488487244, |
| "kl": 0.004197513037070166, |
| "learning_rate": 3.9789473684210525e-06, |
| "loss": -0.0001, |
| "step": 244 |
| }, |
| { |
| "clip_ratio": 0.0005502186249941587, |
| "completion_length": 193.34375, |
| "epoch": 0.03345166575641726, |
| "grad_norm": 0.04852650314569473, |
| "kl": 0.0035738514889089856, |
| "learning_rate": 3.9736842105263165e-06, |
| "loss": 0.0001, |
| "num_tokens": 734035.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 245 |
| }, |
| { |
| "clip_ratio": 0.00011322463979013264, |
| "epoch": 0.03358820316766794, |
| "grad_norm": 0.04901989549398422, |
| "kl": 0.0036475551678449847, |
| "learning_rate": 3.968421052631579e-06, |
| "loss": -0.0002, |
| "step": 246 |
| }, |
| { |
| "clip_ratio": 0.00029761905898340046, |
| "epoch": 0.033724740578918624, |
| "grad_norm": 0.04869365692138672, |
| "kl": 0.0037203980718913954, |
| "learning_rate": 3.963157894736843e-06, |
| "loss": -0.0001, |
| "step": 247 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03386127799016931, |
| "grad_norm": 0.04897074028849602, |
| "kl": 0.0037638081375916954, |
| "learning_rate": 3.957894736842106e-06, |
| "loss": -0.0002, |
| "step": 248 |
| }, |
| { |
| "clip_ratio": 0.0010135627671843395, |
| "completion_length": 158.84375, |
| "epoch": 0.03399781540141999, |
| "grad_norm": 0.036369990557432175, |
| "kl": 0.004446192982868524, |
| "learning_rate": 3.952631578947368e-06, |
| "loss": -0.0, |
| "num_tokens": 747138.0, |
| "reward": 0.7718750238418579, |
| "reward_std": 0.18370190262794495, |
| "rewards/reward_fn": 0.7718750238418579, |
| "step": 249 |
| }, |
| { |
| "clip_ratio": 0.0008063142449827865, |
| "epoch": 0.034134352812670674, |
| "grad_norm": 0.03621983900666237, |
| "kl": 0.004339989216532558, |
| "learning_rate": 3.947368421052632e-06, |
| "loss": 0.0001, |
| "step": 250 |
| }, |
| { |
| "clip_ratio": 0.0008642350876471028, |
| "epoch": 0.03427089022392135, |
| "grad_norm": 0.03663729131221771, |
| "kl": 0.004259787187038455, |
| "learning_rate": 3.942105263157895e-06, |
| "loss": -0.0, |
| "step": 251 |
| }, |
| { |
| "clip_ratio": 0.0007387330842902884, |
| "epoch": 0.03440742763517204, |
| "grad_norm": 0.03616287186741829, |
| "kl": 0.004657330282498151, |
| "learning_rate": 3.936842105263159e-06, |
| "loss": -0.0001, |
| "step": 252 |
| }, |
| { |
| "clip_ratio": 0.00031261908588930964, |
| "completion_length": 188.53125, |
| "epoch": 0.03454396504642272, |
| "grad_norm": 0.03885746747255325, |
| "kl": 0.004843563110625837, |
| "learning_rate": 3.931578947368421e-06, |
| "loss": 0.0002, |
| "num_tokens": 760543.0, |
| "reward": 0.6906249523162842, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.6906249523162842, |
| "step": 253 |
| }, |
| { |
| "clip_ratio": 0.0001220703125, |
| "epoch": 0.0346805024576734, |
| "grad_norm": 0.03930497542023659, |
| "kl": 0.005004588409065036, |
| "learning_rate": 3.926315789473685e-06, |
| "loss": 0.0001, |
| "step": 254 |
| }, |
| { |
| "clip_ratio": 0.0008644816552987322, |
| "epoch": 0.034817039868924086, |
| "grad_norm": 0.043471258133649826, |
| "kl": 0.005154007722012466, |
| "learning_rate": 3.921052631578947e-06, |
| "loss": 0.0001, |
| "step": 255 |
| }, |
| { |
| "clip_ratio": 0.0001220703125, |
| "epoch": 0.034953577280174765, |
| "grad_norm": 0.04276250675320625, |
| "kl": 0.005200162773689954, |
| "learning_rate": 3.9157894736842104e-06, |
| "loss": -0.0, |
| "step": 256 |
| }, |
| { |
| "clip_ratio": 0.00013297871919348836, |
| "completion_length": 211.90625, |
| "epoch": 0.03509011469142545, |
| "grad_norm": 0.035579003393650055, |
| "kl": 0.005733902646170463, |
| "learning_rate": 3.9105263157894744e-06, |
| "loss": 0.0001, |
| "num_tokens": 774600.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 257 |
| }, |
| { |
| "clip_ratio": 0.0003613429289544001, |
| "epoch": 0.035226652102676136, |
| "grad_norm": 0.0358058400452137, |
| "kl": 0.005903951641812455, |
| "learning_rate": 3.905263157894737e-06, |
| "loss": 0.0, |
| "step": 258 |
| }, |
| { |
| "clip_ratio": 0.000147405662573874, |
| "epoch": 0.035363189513926814, |
| "grad_norm": 0.03550832346081734, |
| "kl": 0.006151054822112201, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": 0.0, |
| "step": 259 |
| }, |
| { |
| "clip_ratio": 0.000586163077969104, |
| "epoch": 0.0354997269251775, |
| "grad_norm": 0.03602401167154312, |
| "kl": 0.006195991249114741, |
| "learning_rate": 3.894736842105263e-06, |
| "loss": -0.0001, |
| "step": 260 |
| }, |
| { |
| "clip_ratio": 0.0005947442696196958, |
| "completion_length": 170.28125, |
| "epoch": 0.035636264336428185, |
| "grad_norm": 0.043325502425432205, |
| "kl": 0.006852686783531681, |
| "learning_rate": 3.889473684210527e-06, |
| "loss": 0.0001, |
| "num_tokens": 787405.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 261 |
| }, |
| { |
| "clip_ratio": 0.0011571143404580653, |
| "epoch": 0.03577280174767886, |
| "grad_norm": 0.044165611267089844, |
| "kl": 0.006842119590146467, |
| "learning_rate": 3.884210526315789e-06, |
| "loss": 0.0001, |
| "step": 262 |
| }, |
| { |
| "clip_ratio": 0.00016108246927615255, |
| "epoch": 0.03590933915892955, |
| "grad_norm": 0.04308279976248741, |
| "kl": 0.00680348261812469, |
| "learning_rate": 3.878947368421053e-06, |
| "loss": -0.0001, |
| "step": 263 |
| }, |
| { |
| "clip_ratio": 0.0007333645335165784, |
| "epoch": 0.03604587657018023, |
| "grad_norm": 0.04286092147231102, |
| "kl": 0.006871012526971754, |
| "learning_rate": 3.873684210526316e-06, |
| "loss": 0.0, |
| "step": 264 |
| }, |
| { |
| "clip_ratio": 0.0011716574590536766, |
| "completion_length": 187.8125, |
| "epoch": 0.03618241398143091, |
| "grad_norm": 0.04439049959182739, |
| "kl": 0.006948054251552094, |
| "learning_rate": 3.868421052631579e-06, |
| "loss": 0.0003, |
| "num_tokens": 800575.0, |
| "reward": 0.6312500238418579, |
| "reward_std": 0.23749998211860657, |
| "rewards/reward_fn": 0.6312500238418579, |
| "step": 265 |
| }, |
| { |
| "clip_ratio": 0.00022008129599271342, |
| "epoch": 0.0363189513926816, |
| "grad_norm": 0.04327835515141487, |
| "kl": 0.006532188643177506, |
| "learning_rate": 3.863157894736843e-06, |
| "loss": -0.0, |
| "step": 266 |
| }, |
| { |
| "clip_ratio": 0.0006383293439284898, |
| "epoch": 0.036455488803932276, |
| "grad_norm": 0.04287095367908478, |
| "kl": 0.006718571348756086, |
| "learning_rate": 3.857894736842105e-06, |
| "loss": -0.0001, |
| "step": 267 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03659202621518296, |
| "grad_norm": 0.04293559491634369, |
| "kl": 0.006709258093906101, |
| "learning_rate": 3.852631578947369e-06, |
| "loss": -0.0001, |
| "step": 268 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 195.84375, |
| "epoch": 0.03672856362643364, |
| "grad_norm": 0.029919667169451714, |
| "kl": 0.006744440222973935, |
| "learning_rate": 3.8473684210526316e-06, |
| "loss": -0.0, |
| "num_tokens": 813758.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 269 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.036865101037684325, |
| "grad_norm": 0.02985461801290512, |
| "kl": 0.006855557934613898, |
| "learning_rate": 3.842105263157895e-06, |
| "loss": -0.0001, |
| "step": 270 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03700163844893501, |
| "grad_norm": 0.02994859404861927, |
| "kl": 0.006910524287377484, |
| "learning_rate": 3.836842105263158e-06, |
| "loss": 0.0001, |
| "step": 271 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.03713817586018569, |
| "grad_norm": 0.02996179461479187, |
| "kl": 0.006799034192226827, |
| "learning_rate": 3.831578947368421e-06, |
| "loss": -0.0001, |
| "step": 272 |
| }, |
| { |
| "clip_ratio": 0.0005878518059034832, |
| "completion_length": 193.21875, |
| "epoch": 0.037274713271436374, |
| "grad_norm": 0.040982622653245926, |
| "kl": 0.005112638631544542, |
| "learning_rate": 3.826315789473685e-06, |
| "loss": -0.0001, |
| "num_tokens": 827521.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 273 |
| }, |
| { |
| "clip_ratio": 0.0005884921629331075, |
| "epoch": 0.03741125068268706, |
| "grad_norm": 0.04164090380072594, |
| "kl": 0.005006878869608045, |
| "learning_rate": 3.821052631578947e-06, |
| "loss": 0.0001, |
| "step": 274 |
| }, |
| { |
| "clip_ratio": 0.0005556777541642077, |
| "epoch": 0.03754778809393774, |
| "grad_norm": 0.04127899557352066, |
| "kl": 0.0049030700902221724, |
| "learning_rate": 3.815789473684211e-06, |
| "loss": -0.0004, |
| "step": 275 |
| }, |
| { |
| "clip_ratio": 0.0007842327322578058, |
| "epoch": 0.037684325505188423, |
| "grad_norm": 0.041350048035383224, |
| "kl": 0.005050874984590337, |
| "learning_rate": 3.810526315789474e-06, |
| "loss": -0.0003, |
| "step": 276 |
| }, |
| { |
| "clip_ratio": 0.0006287864307523705, |
| "completion_length": 196.46875, |
| "epoch": 0.0378208629164391, |
| "grad_norm": 0.048398833721876144, |
| "kl": 0.005731382880185265, |
| "learning_rate": 3.805263157894737e-06, |
| "loss": 0.0001, |
| "num_tokens": 841616.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 277 |
| }, |
| { |
| "clip_ratio": 0.00019054877338930964, |
| "epoch": 0.03795740032768979, |
| "grad_norm": 0.04829626902937889, |
| "kl": 0.005529685382498428, |
| "learning_rate": 3.8000000000000005e-06, |
| "loss": -0.0002, |
| "step": 278 |
| }, |
| { |
| "clip_ratio": 0.00044461380457505584, |
| "epoch": 0.03809393773894047, |
| "grad_norm": 0.04743318632245064, |
| "kl": 0.005668424419127405, |
| "learning_rate": 3.794736842105263e-06, |
| "loss": -0.0001, |
| "step": 279 |
| }, |
| { |
| "clip_ratio": 0.0005434168851934373, |
| "epoch": 0.03823047515019115, |
| "grad_norm": 0.048726946115493774, |
| "kl": 0.00560217464226298, |
| "learning_rate": 3.789473684210527e-06, |
| "loss": -0.0004, |
| "step": 280 |
| }, |
| { |
| "clip_ratio": 0.0008525320590706542, |
| "completion_length": 193.5, |
| "epoch": 0.038367012561441836, |
| "grad_norm": 0.047472480684518814, |
| "kl": 0.006455096743593458, |
| "learning_rate": 3.7842105263157895e-06, |
| "loss": 0.0002, |
| "num_tokens": 855160.0, |
| "reward": 0.71875, |
| "reward_std": 0.30735570192337036, |
| "rewards/reward_fn": 0.71875, |
| "step": 281 |
| }, |
| { |
| "clip_ratio": 0.0008893540798453614, |
| "epoch": 0.038503549972692515, |
| "grad_norm": 0.047954004257917404, |
| "kl": 0.006647233945841435, |
| "learning_rate": 3.778947368421053e-06, |
| "loss": 0.0002, |
| "step": 282 |
| }, |
| { |
| "clip_ratio": 0.0007021718411124311, |
| "epoch": 0.0386400873839432, |
| "grad_norm": 0.04781438782811165, |
| "kl": 0.006593440702999942, |
| "learning_rate": 3.773684210526316e-06, |
| "loss": 0.0, |
| "step": 283 |
| }, |
| { |
| "clip_ratio": 0.0005247658991720527, |
| "epoch": 0.038776624795193886, |
| "grad_norm": 0.047404695302248, |
| "kl": 0.00645634390093619, |
| "learning_rate": 3.768421052631579e-06, |
| "loss": 0.0, |
| "step": 284 |
| }, |
| { |
| "clip_ratio": 0.0003453038807492703, |
| "completion_length": 185.8125, |
| "epoch": 0.038913162206444564, |
| "grad_norm": 0.0490223653614521, |
| "kl": 0.006790038540202659, |
| "learning_rate": 3.7631578947368426e-06, |
| "loss": -0.0001, |
| "num_tokens": 868494.0, |
| "reward": 0.71875, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.71875, |
| "step": 285 |
| }, |
| { |
| "clip_ratio": 0.00042671697156038135, |
| "epoch": 0.03904969961769525, |
| "grad_norm": 0.048411138355731964, |
| "kl": 0.0067207906395196915, |
| "learning_rate": 3.7578947368421053e-06, |
| "loss": 0.0001, |
| "step": 286 |
| }, |
| { |
| "clip_ratio": 0.00017265194037463516, |
| "epoch": 0.03918623702894593, |
| "grad_norm": 0.04938926175236702, |
| "kl": 0.006590324410353787, |
| "learning_rate": 3.752631578947369e-06, |
| "loss": -0.0003, |
| "step": 287 |
| }, |
| { |
| "clip_ratio": 0.00042671697156038135, |
| "epoch": 0.03932277444019661, |
| "grad_norm": 0.048639941960573196, |
| "kl": 0.006757089176971931, |
| "learning_rate": 3.7473684210526317e-06, |
| "loss": -0.0, |
| "step": 288 |
| }, |
| { |
| "clip_ratio": 0.0004049283015774563, |
| "completion_length": 200.5625, |
| "epoch": 0.0394593118514473, |
| "grad_norm": 0.03730582445859909, |
| "kl": 0.0059931142459390685, |
| "learning_rate": 3.7421052631578953e-06, |
| "loss": 0.0, |
| "num_tokens": 882000.0, |
| "reward": 0.578125, |
| "reward_std": 0.25110572576522827, |
| "rewards/reward_fn": 0.578125, |
| "step": 289 |
| }, |
| { |
| "clip_ratio": 0.001011994703731034, |
| "epoch": 0.03959584926269798, |
| "grad_norm": 0.03750862926244736, |
| "kl": 0.006054735436919145, |
| "learning_rate": 3.736842105263158e-06, |
| "loss": 0.0001, |
| "step": 290 |
| }, |
| { |
| "clip_ratio": 0.00011617100244620815, |
| "epoch": 0.03973238667394866, |
| "grad_norm": 0.03716864436864853, |
| "kl": 0.006095452328736428, |
| "learning_rate": 3.7315789473684216e-06, |
| "loss": 0.0001, |
| "step": 291 |
| }, |
| { |
| "clip_ratio": 0.00028565472894115373, |
| "epoch": 0.03986892408519935, |
| "grad_norm": 0.037398044019937515, |
| "kl": 0.006145935753011145, |
| "learning_rate": 3.7263157894736848e-06, |
| "loss": -0.0, |
| "step": 292 |
| }, |
| { |
| "clip_ratio": 0.0006079878221498802, |
| "completion_length": 168.125, |
| "epoch": 0.040005461496450026, |
| "grad_norm": 0.0302655640989542, |
| "kl": 0.0076763516444771085, |
| "learning_rate": 3.7210526315789475e-06, |
| "loss": 0.0001, |
| "num_tokens": 894652.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 293 |
| }, |
| { |
| "clip_ratio": 0.0006981320475460961, |
| "epoch": 0.04014199890770071, |
| "grad_norm": 0.030427517369389534, |
| "kl": 0.007548757799668238, |
| "learning_rate": 3.715789473684211e-06, |
| "loss": 0.0001, |
| "step": 294 |
| }, |
| { |
| "clip_ratio": 0.0001502403902122751, |
| "epoch": 0.04027853631895139, |
| "grad_norm": 0.030331620946526527, |
| "kl": 0.007952025051054079, |
| "learning_rate": 3.710526315789474e-06, |
| "loss": -0.0, |
| "step": 295 |
| }, |
| { |
| "clip_ratio": 0.00039825627754908055, |
| "epoch": 0.040415073730202075, |
| "grad_norm": 0.03000173531472683, |
| "kl": 0.007717708278505597, |
| "learning_rate": 3.7052631578947374e-06, |
| "loss": -0.0001, |
| "step": 296 |
| }, |
| { |
| "clip_ratio": 0.0004486169636948034, |
| "completion_length": 191.5625, |
| "epoch": 0.04055161114145276, |
| "grad_norm": 0.045223742723464966, |
| "kl": 0.005729781052650651, |
| "learning_rate": 3.7e-06, |
| "loss": -0.0001, |
| "num_tokens": 908290.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 297 |
| }, |
| { |
| "clip_ratio": 0.00043309624743415043, |
| "epoch": 0.04068814855270344, |
| "grad_norm": 0.04540492221713066, |
| "kl": 0.0059954764728900045, |
| "learning_rate": 3.6947368421052637e-06, |
| "loss": -0.0002, |
| "step": 298 |
| }, |
| { |
| "clip_ratio": 0.00035666103212861344, |
| "epoch": 0.040824685963954124, |
| "grad_norm": 0.044514089822769165, |
| "kl": 0.00598997191991657, |
| "learning_rate": 3.6894736842105265e-06, |
| "loss": -0.0003, |
| "step": 299 |
| }, |
| { |
| "clip_ratio": 0.0002594731267890893, |
| "epoch": 0.0409612233752048, |
| "grad_norm": 0.0427105650305748, |
| "kl": 0.00601626500065322, |
| "learning_rate": 3.6842105263157896e-06, |
| "loss": -0.0004, |
| "step": 300 |
| }, |
| { |
| "clip_ratio": 0.0001502403902122751, |
| "completion_length": 177.65625, |
| "epoch": 0.04109776078645549, |
| "grad_norm": 0.0303980503231287, |
| "kl": 0.006173621120979078, |
| "learning_rate": 3.6789473684210532e-06, |
| "loss": 0.0002, |
| "num_tokens": 921615.0, |
| "reward": 0.859375, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.859375, |
| "step": 301 |
| }, |
| { |
| "clip_ratio": 0.0005745660164393485, |
| "epoch": 0.04123429819770617, |
| "grad_norm": 0.030963070690631866, |
| "kl": 0.006070771494705696, |
| "learning_rate": 3.673684210526316e-06, |
| "loss": 0.0002, |
| "step": 302 |
| }, |
| { |
| "clip_ratio": 0.00029301163158379495, |
| "epoch": 0.04137083560895685, |
| "grad_norm": 0.03128093108534813, |
| "kl": 0.006251918799534906, |
| "learning_rate": 3.6684210526315796e-06, |
| "loss": 0.0001, |
| "step": 303 |
| }, |
| { |
| "clip_ratio": 0.0001509661815362051, |
| "epoch": 0.04150737302020754, |
| "grad_norm": 0.03127000853419304, |
| "kl": 0.006164673512103036, |
| "learning_rate": 3.6631578947368423e-06, |
| "loss": 0.0, |
| "step": 304 |
| }, |
| { |
| "clip_ratio": 0.00013354701513890177, |
| "completion_length": 155.1875, |
| "epoch": 0.04164391043145822, |
| "grad_norm": 0.018649565055966377, |
| "kl": 0.00768783222156344, |
| "learning_rate": 3.657894736842106e-06, |
| "loss": 0.0001, |
| "num_tokens": 933277.0, |
| "reward": 0.971875011920929, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.971875011920929, |
| "step": 305 |
| }, |
| { |
| "clip_ratio": 0.00020559210679493845, |
| "epoch": 0.0417804478427089, |
| "grad_norm": 0.01852833852171898, |
| "kl": 0.007686922417633468, |
| "learning_rate": 3.6526315789473686e-06, |
| "loss": 0.0, |
| "step": 306 |
| }, |
| { |
| "clip_ratio": 0.00013354701513890177, |
| "epoch": 0.041916985253959586, |
| "grad_norm": 0.018789473921060562, |
| "kl": 0.007915814974694513, |
| "learning_rate": 3.6473684210526318e-06, |
| "loss": 0.0, |
| "step": 307 |
| }, |
| { |
| "clip_ratio": 0.00033634524152148515, |
| "epoch": 0.042053522665210265, |
| "grad_norm": 0.018418900668621063, |
| "kl": 0.007627661438164068, |
| "learning_rate": 3.642105263157895e-06, |
| "loss": -0.0, |
| "step": 308 |
| }, |
| { |
| "clip_ratio": 0.0005952381179668009, |
| "completion_length": 166.9375, |
| "epoch": 0.04219006007646095, |
| "grad_norm": 0.05598299950361252, |
| "kl": 0.008668451067933347, |
| "learning_rate": 3.636842105263158e-06, |
| "loss": -0.0, |
| "num_tokens": 945799.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 309 |
| }, |
| { |
| "clip_ratio": 0.0014606260519940406, |
| "epoch": 0.042326597487711635, |
| "grad_norm": 0.05640915036201477, |
| "kl": 0.008580744928622153, |
| "learning_rate": 3.6315789473684217e-06, |
| "loss": 0.0002, |
| "step": 310 |
| }, |
| { |
| "clip_ratio": 0.0007385867356788367, |
| "epoch": 0.042463134898962314, |
| "grad_norm": 0.0538216196000576, |
| "kl": 0.008697011238837149, |
| "learning_rate": 3.6263157894736844e-06, |
| "loss": -0.0001, |
| "step": 311 |
| }, |
| { |
| "clip_ratio": 0.0008078231476247311, |
| "epoch": 0.042599672310213, |
| "grad_norm": 0.054313480854034424, |
| "kl": 0.009035873423272278, |
| "learning_rate": 3.621052631578948e-06, |
| "loss": -0.0, |
| "step": 312 |
| }, |
| { |
| "clip_ratio": 0.00042422240949235857, |
| "completion_length": 145.75, |
| "epoch": 0.04273620972146368, |
| "grad_norm": 0.027714770287275314, |
| "kl": 0.00796969021030236, |
| "learning_rate": 3.6157894736842108e-06, |
| "loss": 0.0001, |
| "num_tokens": 957375.0, |
| "reward": 0.4375000298023224, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.4375000298023224, |
| "step": 313 |
| }, |
| { |
| "clip_ratio": 0.00031646378920413554, |
| "epoch": 0.04287274713271436, |
| "grad_norm": 0.027388686314225197, |
| "kl": 0.008329837735800538, |
| "learning_rate": 3.610526315789474e-06, |
| "loss": 0.0, |
| "step": 314 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04300928454396505, |
| "grad_norm": 0.02774704247713089, |
| "kl": 0.008337227794982027, |
| "learning_rate": 3.605263157894737e-06, |
| "loss": 0.0, |
| "step": 315 |
| }, |
| { |
| "clip_ratio": 0.0003688146098284051, |
| "epoch": 0.04314582195521573, |
| "grad_norm": 0.02776927500963211, |
| "kl": 0.008564134819607716, |
| "learning_rate": 3.6000000000000003e-06, |
| "loss": -0.0, |
| "step": 316 |
| }, |
| { |
| "clip_ratio": 0.0003649798163678497, |
| "completion_length": 171.125, |
| "epoch": 0.04328235936646641, |
| "grad_norm": 0.048168253153562546, |
| "kl": 0.009050727443536744, |
| "learning_rate": 3.5947368421052634e-06, |
| "loss": 0.0003, |
| "num_tokens": 970555.0, |
| "reward": 0.9125000238418579, |
| "reward_std": 0.17499999701976776, |
| "rewards/reward_fn": 0.9125000238418579, |
| "step": 317 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0434188967777171, |
| "grad_norm": 0.04662594571709633, |
| "kl": 0.009075636109628249, |
| "learning_rate": 3.5894736842105266e-06, |
| "loss": 0.0002, |
| "step": 318 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.043555434188967776, |
| "grad_norm": 0.045846883207559586, |
| "kl": 0.008941752610553522, |
| "learning_rate": 3.58421052631579e-06, |
| "loss": -0.0001, |
| "step": 319 |
| }, |
| { |
| "clip_ratio": 0.00033160162274725735, |
| "epoch": 0.04369197160021846, |
| "grad_norm": 0.04646976292133331, |
| "kl": 0.009185112408886198, |
| "learning_rate": 3.578947368421053e-06, |
| "loss": -0.0, |
| "step": 320 |
| }, |
| { |
| "clip_ratio": 0.0009399811024195515, |
| "completion_length": 186.03125, |
| "epoch": 0.04382850901146914, |
| "grad_norm": 0.04080897942185402, |
| "kl": 0.009395269502419978, |
| "learning_rate": 3.5736842105263157e-06, |
| "loss": 0.0001, |
| "num_tokens": 983896.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 321 |
| }, |
| { |
| "clip_ratio": 0.0008206058701034635, |
| "epoch": 0.043965046422719825, |
| "grad_norm": 0.04161817207932472, |
| "kl": 0.00910995878803078, |
| "learning_rate": 3.5684210526315792e-06, |
| "loss": 0.0001, |
| "step": 322 |
| }, |
| { |
| "clip_ratio": 0.0006674195901723579, |
| "epoch": 0.04410158383397051, |
| "grad_norm": 0.04027590528130531, |
| "kl": 0.009239041413820814, |
| "learning_rate": 3.5631578947368424e-06, |
| "loss": -0.0, |
| "step": 323 |
| }, |
| { |
| "clip_ratio": 0.0018579070747364312, |
| "epoch": 0.04423812124522119, |
| "grad_norm": 0.04106101021170616, |
| "kl": 0.009226256901456509, |
| "learning_rate": 3.5578947368421056e-06, |
| "loss": -0.0001, |
| "step": 324 |
| }, |
| { |
| "clip_ratio": 0.00016191709437407553, |
| "completion_length": 129.8125, |
| "epoch": 0.044374658656471874, |
| "grad_norm": 0.05279777571558952, |
| "kl": 0.008043430578254629, |
| "learning_rate": 3.5526315789473687e-06, |
| "loss": 0.0004, |
| "num_tokens": 995098.0, |
| "reward": 0.71875, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.71875, |
| "step": 325 |
| }, |
| { |
| "clip_ratio": 0.0008133933879435062, |
| "epoch": 0.04451119606772255, |
| "grad_norm": 0.05448969081044197, |
| "kl": 0.00803122408251511, |
| "learning_rate": 3.5473684210526323e-06, |
| "loss": 0.0002, |
| "step": 326 |
| }, |
| { |
| "clip_ratio": 0.0002741228090599179, |
| "epoch": 0.04464773347897324, |
| "grad_norm": 0.051325391978025436, |
| "kl": 0.008287725548143499, |
| "learning_rate": 3.542105263157895e-06, |
| "loss": 0.0, |
| "step": 327 |
| }, |
| { |
| "clip_ratio": 0.000400130869820714, |
| "epoch": 0.04478427089022392, |
| "grad_norm": 0.05115650221705437, |
| "kl": 0.008330306562129408, |
| "learning_rate": 3.536842105263158e-06, |
| "loss": 0.0, |
| "step": 328 |
| }, |
| { |
| "clip_ratio": 0.0011260724713793024, |
| "completion_length": 169.0625, |
| "epoch": 0.0449208083014746, |
| "grad_norm": 0.057967912405729294, |
| "kl": 0.01103355574014131, |
| "learning_rate": 3.5315789473684214e-06, |
| "loss": 0.0004, |
| "num_tokens": 1007628.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 329 |
| }, |
| { |
| "clip_ratio": 0.0003532530608936213, |
| "epoch": 0.04505734571272529, |
| "grad_norm": 0.05616287142038345, |
| "kl": 0.010974232252920046, |
| "learning_rate": 3.5263157894736846e-06, |
| "loss": 0.0002, |
| "step": 330 |
| }, |
| { |
| "clip_ratio": 0.00044301825255388394, |
| "epoch": 0.04519388312397597, |
| "grad_norm": 0.05720368027687073, |
| "kl": 0.011127974459668621, |
| "learning_rate": 3.5210526315789477e-06, |
| "loss": 0.0001, |
| "step": 331 |
| }, |
| { |
| "clip_ratio": 0.0005052211708971299, |
| "epoch": 0.04533042053522665, |
| "grad_norm": 0.05788964405655861, |
| "kl": 0.011284424290352035, |
| "learning_rate": 3.515789473684211e-06, |
| "loss": -0.0002, |
| "step": 332 |
| }, |
| { |
| "clip_ratio": 0.0006831782375229523, |
| "completion_length": 167.59375, |
| "epoch": 0.045466957946477336, |
| "grad_norm": 0.04411827400326729, |
| "kl": 0.008889463526429608, |
| "learning_rate": 3.510526315789474e-06, |
| "loss": 0.0002, |
| "num_tokens": 1019999.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 333 |
| }, |
| { |
| "clip_ratio": 0.0003221649385523051, |
| "epoch": 0.045603495357728015, |
| "grad_norm": 0.04477844387292862, |
| "kl": 0.008871635320247151, |
| "learning_rate": 3.505263157894737e-06, |
| "loss": -0.0, |
| "step": 334 |
| }, |
| { |
| "clip_ratio": 0.00048663862980902195, |
| "epoch": 0.0457400327689787, |
| "grad_norm": 0.042167555540800095, |
| "kl": 0.008778905357758049, |
| "learning_rate": 3.5e-06, |
| "loss": 0.0, |
| "step": 335 |
| }, |
| { |
| "clip_ratio": 0.0003221649385523051, |
| "epoch": 0.045876570180229385, |
| "grad_norm": 0.0451904833316803, |
| "kl": 0.008919417305151, |
| "learning_rate": 3.4947368421052635e-06, |
| "loss": -0.0001, |
| "step": 336 |
| }, |
| { |
| "clip_ratio": 0.0011595691612455994, |
| "completion_length": 193.21875, |
| "epoch": 0.046013107591480064, |
| "grad_norm": 0.048833779990673065, |
| "kl": 0.008373326862056274, |
| "learning_rate": 3.4894736842105263e-06, |
| "loss": 0.0001, |
| "num_tokens": 1033218.0, |
| "reward": 0.578125, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.578125, |
| "step": 337 |
| }, |
| { |
| "clip_ratio": 0.00036128354258835316, |
| "epoch": 0.04614964500273075, |
| "grad_norm": 0.048138685524463654, |
| "kl": 0.008346186616108753, |
| "learning_rate": 3.48421052631579e-06, |
| "loss": 0.0001, |
| "step": 338 |
| }, |
| { |
| "clip_ratio": 0.0006581510824616998, |
| "epoch": 0.04628618241398143, |
| "grad_norm": 0.048573918640613556, |
| "kl": 0.008154301816830412, |
| "learning_rate": 3.478947368421053e-06, |
| "loss": 0.0002, |
| "step": 339 |
| }, |
| { |
| "clip_ratio": 0.0005721003690268844, |
| "epoch": 0.04642271982523211, |
| "grad_norm": 0.04779474809765816, |
| "kl": 0.008427414650213905, |
| "learning_rate": 3.473684210526316e-06, |
| "loss": -0.0001, |
| "step": 340 |
| }, |
| { |
| "clip_ratio": 0.0006677096389466897, |
| "completion_length": 182.0, |
| "epoch": 0.0465592572364828, |
| "grad_norm": 0.04990622401237488, |
| "kl": 0.008463293423119467, |
| "learning_rate": 3.4684210526315794e-06, |
| "loss": 0.0003, |
| "num_tokens": 1046522.0, |
| "reward": 0.49375003576278687, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.49375003576278687, |
| "step": 341 |
| }, |
| { |
| "clip_ratio": 0.00024055899120867252, |
| "epoch": 0.04669579464773348, |
| "grad_norm": 0.04765518754720688, |
| "kl": 0.008242011550464667, |
| "learning_rate": 3.463157894736842e-06, |
| "loss": 0.0, |
| "step": 342 |
| }, |
| { |
| "clip_ratio": 0.0003520036116242409, |
| "epoch": 0.04683233205898416, |
| "grad_norm": 0.049412552267313004, |
| "kl": 0.00797222027904354, |
| "learning_rate": 3.4578947368421057e-06, |
| "loss": 0.0, |
| "step": 343 |
| }, |
| { |
| "clip_ratio": 0.0006752904300810769, |
| "epoch": 0.04696886947023485, |
| "grad_norm": 0.047082945704460144, |
| "kl": 0.007959613183629699, |
| "learning_rate": 3.4526315789473684e-06, |
| "loss": -0.0002, |
| "step": 344 |
| }, |
| { |
| "clip_ratio": 0.0006411051435861737, |
| "completion_length": 155.875, |
| "epoch": 0.047105406881485526, |
| "grad_norm": 0.035547517240047455, |
| "kl": 0.008352179189387243, |
| "learning_rate": 3.447368421052632e-06, |
| "loss": 0.0003, |
| "num_tokens": 1058174.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 345 |
| }, |
| { |
| "clip_ratio": 0.0006411051435861737, |
| "epoch": 0.04724194429273621, |
| "grad_norm": 0.036628782749176025, |
| "kl": 0.008347121729457285, |
| "learning_rate": 3.4421052631578947e-06, |
| "loss": 0.0002, |
| "step": 346 |
| }, |
| { |
| "clip_ratio": 0.00016276042151730508, |
| "epoch": 0.04737848170398689, |
| "grad_norm": 0.035852521657943726, |
| "kl": 0.008160358331224415, |
| "learning_rate": 3.4368421052631583e-06, |
| "loss": 0.0002, |
| "step": 347 |
| }, |
| { |
| "clip_ratio": 0.0001502403902122751, |
| "epoch": 0.047515019115237575, |
| "grad_norm": 0.03534158319234848, |
| "kl": 0.007882972859079018, |
| "learning_rate": 3.4315789473684215e-06, |
| "loss": -0.0, |
| "step": 348 |
| }, |
| { |
| "clip_ratio": 0.00038622794090770185, |
| "completion_length": 153.65625, |
| "epoch": 0.04765155652648826, |
| "grad_norm": 0.05618682876229286, |
| "kl": 0.007273747964063659, |
| "learning_rate": 3.4263157894736842e-06, |
| "loss": -0.0, |
| "num_tokens": 1070387.0, |
| "reward": 0.35312503576278687, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.35312503576278687, |
| "step": 349 |
| }, |
| { |
| "clip_ratio": 0.00048652967961970717, |
| "epoch": 0.04778809393773894, |
| "grad_norm": 0.0554373674094677, |
| "kl": 0.007289341767318547, |
| "learning_rate": 3.421052631578948e-06, |
| "loss": -0.0001, |
| "step": 350 |
| }, |
| { |
| "clip_ratio": 0.0005627816135529429, |
| "epoch": 0.047924631348989624, |
| "grad_norm": 0.0532095730304718, |
| "kl": 0.0073152392214979045, |
| "learning_rate": 3.4157894736842106e-06, |
| "loss": -0.0002, |
| "step": 351 |
| }, |
| { |
| "clip_ratio": 0.0008069222385529429, |
| "epoch": 0.0480611687602403, |
| "grad_norm": 0.05216081812977791, |
| "kl": 0.00747527347994037, |
| "learning_rate": 3.410526315789474e-06, |
| "loss": -0.0005, |
| "step": 352 |
| }, |
| { |
| "clip_ratio": 0.000756516121327877, |
| "completion_length": 134.71875, |
| "epoch": 0.04819770617149099, |
| "grad_norm": 0.05914979800581932, |
| "kl": 0.009078809467609972, |
| "learning_rate": 3.405263157894737e-06, |
| "loss": 0.0002, |
| "num_tokens": 1082098.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.578125, |
| "step": 353 |
| }, |
| { |
| "clip_ratio": 0.00011574073869269341, |
| "epoch": 0.04833424358274167, |
| "grad_norm": 0.05772961676120758, |
| "kl": 0.00910463748004986, |
| "learning_rate": 3.4000000000000005e-06, |
| "loss": -0.0, |
| "step": 354 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.04847078099399235, |
| "grad_norm": 0.058861494064331055, |
| "kl": 0.008914725520298816, |
| "learning_rate": 3.3947368421052636e-06, |
| "loss": -0.0, |
| "step": 355 |
| }, |
| { |
| "clip_ratio": 0.00019290123600512743, |
| "epoch": 0.04860731840524304, |
| "grad_norm": 0.0598762147128582, |
| "kl": 0.008906295384804253, |
| "learning_rate": 3.3894736842105264e-06, |
| "loss": -0.0002, |
| "step": 356 |
| }, |
| { |
| "clip_ratio": 0.0016233276692219079, |
| "completion_length": 155.75, |
| "epoch": 0.04874385581649372, |
| "grad_norm": 0.0463152751326561, |
| "kl": 0.006661417035502382, |
| "learning_rate": 3.38421052631579e-06, |
| "loss": 0.0001, |
| "num_tokens": 1094526.0, |
| "reward": 0.5500000715255737, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.5500000715255737, |
| "step": 357 |
| }, |
| { |
| "clip_ratio": 0.000750684121157974, |
| "epoch": 0.0488803932277444, |
| "grad_norm": 0.045359767973423004, |
| "kl": 0.0063057993102120236, |
| "learning_rate": 3.3789473684210527e-06, |
| "loss": 0.0002, |
| "step": 358 |
| }, |
| { |
| "clip_ratio": 0.0014823784586042166, |
| "epoch": 0.049016930638995086, |
| "grad_norm": 0.04643021151423454, |
| "kl": 0.006138000971986912, |
| "learning_rate": 3.3736842105263163e-06, |
| "loss": -0.0002, |
| "step": 359 |
| }, |
| { |
| "clip_ratio": 0.0015740015369374305, |
| "epoch": 0.049153468050245765, |
| "grad_norm": 0.04552228003740311, |
| "kl": 0.006226084668014664, |
| "learning_rate": 3.368421052631579e-06, |
| "loss": -0.0003, |
| "step": 360 |
| }, |
| { |
| "clip_ratio": 0.0002883385823224671, |
| "completion_length": 159.3125, |
| "epoch": 0.04929000546149645, |
| "grad_norm": 0.04084038361907005, |
| "kl": 0.0071210484275070485, |
| "learning_rate": 3.3631578947368426e-06, |
| "loss": -0.0001, |
| "num_tokens": 1106860.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 361 |
| }, |
| { |
| "clip_ratio": 0.0006467864877777174, |
| "epoch": 0.049426542872747135, |
| "grad_norm": 0.04093595966696739, |
| "kl": 0.00718199376569828, |
| "learning_rate": 3.3578947368421054e-06, |
| "loss": 0.0001, |
| "step": 362 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.049563080283997814, |
| "grad_norm": 0.040467701852321625, |
| "kl": 0.006776689384423662, |
| "learning_rate": 3.3526315789473685e-06, |
| "loss": -0.0002, |
| "step": 363 |
| }, |
| { |
| "clip_ratio": 0.0005696260050171986, |
| "epoch": 0.0496996176952485, |
| "grad_norm": 0.04037117958068848, |
| "kl": 0.006817667483119294, |
| "learning_rate": 3.347368421052632e-06, |
| "loss": -0.0002, |
| "step": 364 |
| }, |
| { |
| "clip_ratio": 0.0005642871255986392, |
| "completion_length": 128.71875, |
| "epoch": 0.04983615510649918, |
| "grad_norm": 0.05545398220419884, |
| "kl": 0.00654399098129943, |
| "learning_rate": 3.342105263157895e-06, |
| "loss": 0.0, |
| "num_tokens": 1117979.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.25980761647224426, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 365 |
| }, |
| { |
| "clip_ratio": 0.00016891892300918698, |
| "epoch": 0.04997269251774986, |
| "grad_norm": 0.055229347199201584, |
| "kl": 0.00643142945409636, |
| "learning_rate": 3.3368421052631584e-06, |
| "loss": -0.0001, |
| "step": 366 |
| }, |
| { |
| "clip_ratio": 0.00016891892300918698, |
| "epoch": 0.05010922992900055, |
| "grad_norm": 0.05497383326292038, |
| "kl": 0.006414015544578433, |
| "learning_rate": 3.331578947368421e-06, |
| "loss": -0.0001, |
| "step": 367 |
| }, |
| { |
| "clip_ratio": 0.0004360129532869905, |
| "epoch": 0.05024576734025123, |
| "grad_norm": 0.05532035231590271, |
| "kl": 0.00638128952414263, |
| "learning_rate": 3.3263157894736848e-06, |
| "loss": -0.0, |
| "step": 368 |
| }, |
| { |
| "clip_ratio": 0.000155472633196041, |
| "completion_length": 169.25, |
| "epoch": 0.05038230475150191, |
| "grad_norm": 0.035683825612068176, |
| "kl": 0.006187475075421389, |
| "learning_rate": 3.3210526315789475e-06, |
| "loss": 0.0, |
| "num_tokens": 1130827.0, |
| "reward": 0.49375003576278687, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.49375003576278687, |
| "step": 369 |
| }, |
| { |
| "clip_ratio": 0.0006639838975388557, |
| "epoch": 0.0505188421627526, |
| "grad_norm": 0.034932296723127365, |
| "kl": 0.0062468878022627905, |
| "learning_rate": 3.3157894736842107e-06, |
| "loss": 0.0001, |
| "step": 370 |
| }, |
| { |
| "clip_ratio": 0.0004434910661075264, |
| "epoch": 0.050655379574003276, |
| "grad_norm": 0.03588123992085457, |
| "kl": 0.006058048253180459, |
| "learning_rate": 3.310526315789474e-06, |
| "loss": 0.0001, |
| "step": 371 |
| }, |
| { |
| "clip_ratio": 0.0006882205052534118, |
| "epoch": 0.05079191698525396, |
| "grad_norm": 0.036134492605924606, |
| "kl": 0.006105918495450169, |
| "learning_rate": 3.305263157894737e-06, |
| "loss": -0.0, |
| "step": 372 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 143.1875, |
| "epoch": 0.05092845439650464, |
| "grad_norm": 0.03361711651086807, |
| "kl": 0.006160892604384571, |
| "learning_rate": 3.3000000000000006e-06, |
| "loss": 0.0001, |
| "num_tokens": 1142649.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 373 |
| }, |
| { |
| "clip_ratio": 0.0004214266227791086, |
| "epoch": 0.051064991807755325, |
| "grad_norm": 0.033134106546640396, |
| "kl": 0.005984412040561438, |
| "learning_rate": 3.2947368421052633e-06, |
| "loss": 0.0002, |
| "step": 374 |
| }, |
| { |
| "clip_ratio": 0.0003967252268921584, |
| "epoch": 0.05120152921900601, |
| "grad_norm": 0.032204654067754745, |
| "kl": 0.006141213176306337, |
| "learning_rate": 3.289473684210527e-06, |
| "loss": 0.0001, |
| "step": 375 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05133806663025669, |
| "grad_norm": 0.03209593519568443, |
| "kl": 0.006134091017884202, |
| "learning_rate": 3.2842105263157897e-06, |
| "loss": -0.0001, |
| "step": 376 |
| }, |
| { |
| "clip_ratio": 0.0005507410241989419, |
| "completion_length": 175.15625, |
| "epoch": 0.051474604041507374, |
| "grad_norm": 0.055287301540374756, |
| "kl": 0.004484998222324066, |
| "learning_rate": 3.278947368421053e-06, |
| "loss": -0.0001, |
| "num_tokens": 1156362.0, |
| "reward": 0.4937500059604645, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.4937500059604645, |
| "step": 377 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05161114145275805, |
| "grad_norm": 0.055734410881996155, |
| "kl": 0.004466786558623426, |
| "learning_rate": 3.273684210526316e-06, |
| "loss": -0.0003, |
| "step": 378 |
| }, |
| { |
| "clip_ratio": 8.632597018731758e-05, |
| "epoch": 0.05174767886400874, |
| "grad_norm": 0.05478140339255333, |
| "kl": 0.00448507835608325, |
| "learning_rate": 3.268421052631579e-06, |
| "loss": -0.0004, |
| "step": 379 |
| }, |
| { |
| "clip_ratio": 0.0004942235027556308, |
| "epoch": 0.05188421627525942, |
| "grad_norm": 0.05364792421460152, |
| "kl": 0.004345458171883365, |
| "learning_rate": 3.2631578947368423e-06, |
| "loss": -0.0005, |
| "step": 380 |
| }, |
| { |
| "clip_ratio": 0.000700607277394738, |
| "completion_length": 178.625, |
| "epoch": 0.0520207536865101, |
| "grad_norm": 0.04043024033308029, |
| "kl": 0.005427893876913004, |
| "learning_rate": 3.2578947368421055e-06, |
| "loss": 0.0001, |
| "num_tokens": 1168982.0, |
| "reward": 0.7750000357627869, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7750000357627869, |
| "step": 381 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05215729109776079, |
| "grad_norm": 0.04022173956036568, |
| "kl": 0.005587494513747515, |
| "learning_rate": 3.252631578947369e-06, |
| "loss": -0.0002, |
| "step": 382 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05229382850901147, |
| "grad_norm": 0.03905121237039566, |
| "kl": 0.005427953394246288, |
| "learning_rate": 3.247368421052632e-06, |
| "loss": -0.0003, |
| "step": 383 |
| }, |
| { |
| "clip_ratio": 0.00028935185400769114, |
| "epoch": 0.05243036592026215, |
| "grad_norm": 0.039256297051906586, |
| "kl": 0.005416683758085128, |
| "learning_rate": 3.2421052631578945e-06, |
| "loss": -0.0003, |
| "step": 384 |
| }, |
| { |
| "clip_ratio": 0.0008180855948012322, |
| "completion_length": 130.5, |
| "epoch": 0.052566903331512836, |
| "grad_norm": 0.03270651027560234, |
| "kl": 0.0049058933327614795, |
| "learning_rate": 3.236842105263158e-06, |
| "loss": 0.0002, |
| "num_tokens": 1179818.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 385 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.052703440742763515, |
| "grad_norm": 0.03213141858577728, |
| "kl": 0.005080945098598022, |
| "learning_rate": 3.2315789473684213e-06, |
| "loss": 0.0001, |
| "step": 386 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0528399781540142, |
| "grad_norm": 0.03164929896593094, |
| "kl": 0.004898248251265613, |
| "learning_rate": 3.2263157894736845e-06, |
| "loss": -0.0, |
| "step": 387 |
| }, |
| { |
| "clip_ratio": 0.00021853146608918905, |
| "epoch": 0.052976515565264885, |
| "grad_norm": 0.03228931874036789, |
| "kl": 0.0049260632586083375, |
| "learning_rate": 3.2210526315789476e-06, |
| "loss": 0.0001, |
| "step": 388 |
| }, |
| { |
| "clip_ratio": 8.311169949593022e-05, |
| "completion_length": 129.46875, |
| "epoch": 0.053113052976515564, |
| "grad_norm": 0.05108320713043213, |
| "kl": 0.00638336525298655, |
| "learning_rate": 3.215789473684211e-06, |
| "loss": -0.0001, |
| "num_tokens": 1191337.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 389 |
| }, |
| { |
| "clip_ratio": 8.311169949593022e-05, |
| "epoch": 0.05324959038776625, |
| "grad_norm": 0.05068688467144966, |
| "kl": 0.006515413835586514, |
| "learning_rate": 3.210526315789474e-06, |
| "loss": -0.0002, |
| "step": 390 |
| }, |
| { |
| "clip_ratio": 0.00021114865376148373, |
| "epoch": 0.05338612779901693, |
| "grad_norm": 0.05087953060865402, |
| "kl": 0.0066130347331636585, |
| "learning_rate": 3.2052631578947367e-06, |
| "loss": 0.0, |
| "step": 391 |
| }, |
| { |
| "clip_ratio": 0.0009084139674087055, |
| "epoch": 0.05352266521026761, |
| "grad_norm": 0.050263408571481705, |
| "kl": 0.0064137740700971335, |
| "learning_rate": 3.2000000000000003e-06, |
| "loss": -0.0003, |
| "step": 392 |
| }, |
| { |
| "clip_ratio": 0.00022163119865581393, |
| "completion_length": 146.15625, |
| "epoch": 0.0536592026215183, |
| "grad_norm": 0.050062090158462524, |
| "kl": 0.0056292706285603344, |
| "learning_rate": 3.1947368421052634e-06, |
| "loss": 0.0001, |
| "num_tokens": 1203322.0, |
| "reward": 0.831250011920929, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.831250011920929, |
| "step": 393 |
| }, |
| { |
| "clip_ratio": 0.00022163119865581393, |
| "epoch": 0.05379574003276898, |
| "grad_norm": 0.05003751441836357, |
| "kl": 0.005806234854389913, |
| "learning_rate": 3.1894736842105266e-06, |
| "loss": 0.0, |
| "step": 394 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05393227744401966, |
| "grad_norm": 0.05016130208969116, |
| "kl": 0.005803627245768439, |
| "learning_rate": 3.1842105263157898e-06, |
| "loss": -0.0, |
| "step": 395 |
| }, |
| { |
| "clip_ratio": 0.00046577182365581393, |
| "epoch": 0.05406881485527035, |
| "grad_norm": 0.04960909113287926, |
| "kl": 0.005761561438703211, |
| "learning_rate": 3.178947368421053e-06, |
| "loss": -0.0001, |
| "step": 396 |
| }, |
| { |
| "clip_ratio": 0.0005974264786345884, |
| "completion_length": 116.34375, |
| "epoch": 0.054205352266521026, |
| "grad_norm": 0.042336683720350266, |
| "kl": 0.005576848645432619, |
| "learning_rate": 3.173684210526316e-06, |
| "loss": 0.0002, |
| "num_tokens": 1214197.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 397 |
| }, |
| { |
| "clip_ratio": 0.0003876076953019947, |
| "epoch": 0.05434188967777171, |
| "grad_norm": 0.04131317511200905, |
| "kl": 0.005699832778191194, |
| "learning_rate": 3.168421052631579e-06, |
| "loss": 0.0001, |
| "step": 398 |
| }, |
| { |
| "clip_ratio": 0.0003876076953019947, |
| "epoch": 0.05447842708902239, |
| "grad_norm": 0.04251126945018768, |
| "kl": 0.005654203625454102, |
| "learning_rate": 3.1631578947368424e-06, |
| "loss": 0.0001, |
| "step": 399 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.054614964500273075, |
| "grad_norm": 0.04224705696105957, |
| "kl": 0.005721186982555082, |
| "learning_rate": 3.157894736842105e-06, |
| "loss": 0.0, |
| "step": 400 |
| }, |
| { |
| "clip_ratio": 0.00026260505546815693, |
| "completion_length": 131.03125, |
| "epoch": 0.05475150191152376, |
| "grad_norm": 0.07190526276826859, |
| "kl": 0.006595060953259235, |
| "learning_rate": 3.1526315789473688e-06, |
| "loss": -0.0003, |
| "num_tokens": 1225486.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 401 |
| }, |
| { |
| "clip_ratio": 0.00013130252773407847, |
| "epoch": 0.05488803932277444, |
| "grad_norm": 0.07028702646493912, |
| "kl": 0.006788846018025652, |
| "learning_rate": 3.147368421052632e-06, |
| "loss": -0.0004, |
| "step": 402 |
| }, |
| { |
| "clip_ratio": 0.00013130252773407847, |
| "epoch": 0.055024576734025124, |
| "grad_norm": 0.07161487638950348, |
| "kl": 0.006938276626897277, |
| "learning_rate": 3.142105263157895e-06, |
| "loss": -0.0004, |
| "step": 403 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0551611141452758, |
| "grad_norm": 0.06346676498651505, |
| "kl": 0.007136655578506179, |
| "learning_rate": 3.1368421052631582e-06, |
| "loss": -0.0008, |
| "step": 404 |
| }, |
| { |
| "clip_ratio": 0.0008396312041440979, |
| "completion_length": 128.625, |
| "epoch": 0.05529765155652649, |
| "grad_norm": 0.06222056224942207, |
| "kl": 0.00625276272330666, |
| "learning_rate": 3.131578947368421e-06, |
| "loss": -0.0001, |
| "num_tokens": 1236746.0, |
| "reward": 0.4937499761581421, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.4937499761581421, |
| "step": 405 |
| }, |
| { |
| "clip_ratio": 0.0007351997337536886, |
| "epoch": 0.05543418896777717, |
| "grad_norm": 0.06255799531936646, |
| "kl": 0.006171589186124038, |
| "learning_rate": 3.1263157894736846e-06, |
| "loss": -0.0003, |
| "step": 406 |
| }, |
| { |
| "clip_ratio": 0.0010756431001937017, |
| "epoch": 0.05557072637902785, |
| "grad_norm": 0.0618608258664608, |
| "kl": 0.006227073547051987, |
| "learning_rate": 3.1210526315789473e-06, |
| "loss": -0.0003, |
| "step": 407 |
| }, |
| { |
| "clip_ratio": 0.0013540884683607146, |
| "epoch": 0.05570726379027854, |
| "grad_norm": 0.06302416324615479, |
| "kl": 0.006306479932391085, |
| "learning_rate": 3.115789473684211e-06, |
| "loss": -0.0004, |
| "step": 408 |
| }, |
| { |
| "clip_ratio": 0.0003289473825134337, |
| "completion_length": 130.53125, |
| "epoch": 0.05584380120152922, |
| "grad_norm": 0.041258443146944046, |
| "kl": 0.007165696697484236, |
| "learning_rate": 3.1105263157894736e-06, |
| "loss": 0.0, |
| "num_tokens": 1248239.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 409 |
| }, |
| { |
| "clip_ratio": 0.0006030701915733516, |
| "epoch": 0.0559803386127799, |
| "grad_norm": 0.041105739772319794, |
| "kl": 0.007339512674661819, |
| "learning_rate": 3.1052631578947372e-06, |
| "loss": 0.0001, |
| "step": 410 |
| }, |
| { |
| "clip_ratio": 0.0003289473825134337, |
| "epoch": 0.056116876024030586, |
| "grad_norm": 0.04173194244503975, |
| "kl": 0.007270335467183031, |
| "learning_rate": 3.1000000000000004e-06, |
| "loss": -0.0001, |
| "step": 411 |
| }, |
| { |
| "clip_ratio": 0.0003289473825134337, |
| "epoch": 0.056253413435281265, |
| "grad_norm": 0.041046392172575, |
| "kl": 0.007239690006827004, |
| "learning_rate": 3.094736842105263e-06, |
| "loss": -0.0001, |
| "step": 412 |
| }, |
| { |
| "clip_ratio": 0.00012755101488437504, |
| "completion_length": 120.4375, |
| "epoch": 0.05638995084653195, |
| "grad_norm": 0.05379099398851395, |
| "kl": 0.007061240350594744, |
| "learning_rate": 3.0894736842105267e-06, |
| "loss": -0.0, |
| "num_tokens": 1259177.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 413 |
| }, |
| { |
| "clip_ratio": 0.0006713097682222724, |
| "epoch": 0.056526488257782635, |
| "grad_norm": 0.05514014884829521, |
| "kl": 0.006974886076932307, |
| "learning_rate": 3.0842105263157895e-06, |
| "loss": 0.0001, |
| "step": 414 |
| }, |
| { |
| "clip_ratio": 0.0003556532028596848, |
| "epoch": 0.056663025669033314, |
| "grad_norm": 0.05505179986357689, |
| "kl": 0.007155751911341213, |
| "learning_rate": 3.078947368421053e-06, |
| "loss": -0.0002, |
| "step": 415 |
| }, |
| { |
| "clip_ratio": 0.00029289492522366345, |
| "epoch": 0.056799563080284, |
| "grad_norm": 0.05438273027539253, |
| "kl": 0.0074290391930844635, |
| "learning_rate": 3.0736842105263158e-06, |
| "loss": -0.0004, |
| "step": 416 |
| }, |
| { |
| "clip_ratio": 0.00028280544211156666, |
| "completion_length": 123.625, |
| "epoch": 0.05693610049153468, |
| "grad_norm": 0.0459333099424839, |
| "kl": 0.0076482784788822755, |
| "learning_rate": 3.0684210526315794e-06, |
| "loss": 0.0002, |
| "num_tokens": 1270701.0, |
| "reward": 0.578125, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.578125, |
| "step": 417 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05707263790278536, |
| "grad_norm": 0.045840129256248474, |
| "kl": 0.00759352496243082, |
| "learning_rate": 3.0631578947368425e-06, |
| "loss": -0.0001, |
| "step": 418 |
| }, |
| { |
| "clip_ratio": 0.0006578452448593453, |
| "epoch": 0.05720917531403605, |
| "grad_norm": 0.04485239088535309, |
| "kl": 0.007927804057544563, |
| "learning_rate": 3.0578947368421053e-06, |
| "loss": -0.0002, |
| "step": 419 |
| }, |
| { |
| "clip_ratio": 0.00019290123600512743, |
| "epoch": 0.05734571272528673, |
| "grad_norm": 0.046273279935121536, |
| "kl": 0.008006361982552335, |
| "learning_rate": 3.052631578947369e-06, |
| "loss": -0.0002, |
| "step": 420 |
| }, |
| { |
| "clip_ratio": 0.0007452272111549973, |
| "completion_length": 113.65625, |
| "epoch": 0.05748225013653741, |
| "grad_norm": 0.06694253534078598, |
| "kl": 0.010357652165112086, |
| "learning_rate": 3.0473684210526316e-06, |
| "loss": 0.0, |
| "num_tokens": 1281246.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 421 |
| }, |
| { |
| "clip_ratio": 0.0013735106913372874, |
| "epoch": 0.0576187875477881, |
| "grad_norm": 0.06684062629938126, |
| "kl": 0.010819239723787177, |
| "learning_rate": 3.042105263157895e-06, |
| "loss": -0.0, |
| "step": 422 |
| }, |
| { |
| "clip_ratio": 0.00025826445198617876, |
| "epoch": 0.057755324959038776, |
| "grad_norm": 0.0649491474032402, |
| "kl": 0.010554315078479704, |
| "learning_rate": 3.036842105263158e-06, |
| "loss": -0.0003, |
| "step": 423 |
| }, |
| { |
| "clip_ratio": 0.0010093810269609094, |
| "epoch": 0.05789186237028946, |
| "grad_norm": 0.06898830831050873, |
| "kl": 0.01103316882654326, |
| "learning_rate": 3.0315789473684215e-06, |
| "loss": -0.0002, |
| "step": 424 |
| }, |
| { |
| "clip_ratio": 0.0008966522727860138, |
| "completion_length": 121.90625, |
| "epoch": 0.05802839978154014, |
| "grad_norm": 0.056012995541095734, |
| "kl": 0.009156441927189007, |
| "learning_rate": 3.0263157894736843e-06, |
| "loss": 0.0, |
| "num_tokens": 1292007.0, |
| "reward": 0.4937500059604645, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.4937500059604645, |
| "step": 425 |
| }, |
| { |
| "clip_ratio": 0.001370725134620443, |
| "epoch": 0.058164937192790825, |
| "grad_norm": 0.05625171214342117, |
| "kl": 0.009024750004755333, |
| "learning_rate": 3.0210526315789474e-06, |
| "loss": 0.0001, |
| "step": 426 |
| }, |
| { |
| "clip_ratio": 0.001088370059733279, |
| "epoch": 0.05830147460404151, |
| "grad_norm": 0.05657302215695381, |
| "kl": 0.009145595900918124, |
| "learning_rate": 3.015789473684211e-06, |
| "loss": -0.0003, |
| "step": 427 |
| }, |
| { |
| "clip_ratio": 0.0003956322179874405, |
| "epoch": 0.05843801201529219, |
| "grad_norm": 0.05530933290719986, |
| "kl": 0.009006000487715937, |
| "learning_rate": 3.0105263157894737e-06, |
| "loss": -0.0002, |
| "step": 428 |
| }, |
| { |
| "clip_ratio": 0.0003810975467786193, |
| "completion_length": 123.28125, |
| "epoch": 0.058574549426542874, |
| "grad_norm": 0.052457232028245926, |
| "kl": 0.007761141088849399, |
| "learning_rate": 3.0052631578947373e-06, |
| "loss": 0.0001, |
| "num_tokens": 1302804.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 429 |
| }, |
| { |
| "clip_ratio": 0.0002297794126207009, |
| "epoch": 0.05871108683779355, |
| "grad_norm": 0.054180338978767395, |
| "kl": 0.007841004990041256, |
| "learning_rate": 3e-06, |
| "loss": 0.0002, |
| "step": 430 |
| }, |
| { |
| "clip_ratio": 0.0002297794126207009, |
| "epoch": 0.05884762424904424, |
| "grad_norm": 0.052567847073078156, |
| "kl": 0.008006619886145927, |
| "learning_rate": 2.9947368421052637e-06, |
| "loss": 0.0, |
| "step": 431 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.05898416166029492, |
| "grad_norm": 0.052135925740003586, |
| "kl": 0.00781235791509971, |
| "learning_rate": 2.9894736842105264e-06, |
| "loss": -0.0002, |
| "step": 432 |
| }, |
| { |
| "clip_ratio": 0.0012734236079268157, |
| "completion_length": 141.53125, |
| "epoch": 0.0591206990715456, |
| "grad_norm": 0.06543916463851929, |
| "kl": 0.007969716127263382, |
| "learning_rate": 2.9842105263157896e-06, |
| "loss": 0.0001, |
| "num_tokens": 1314101.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.298653781414032, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 433 |
| }, |
| { |
| "clip_ratio": 0.0015249361022142693, |
| "epoch": 0.05925723648279629, |
| "grad_norm": 0.06564782559871674, |
| "kl": 0.007776469879900105, |
| "learning_rate": 2.9789473684210527e-06, |
| "loss": 0.0002, |
| "step": 434 |
| }, |
| { |
| "clip_ratio": 0.0012296376517042518, |
| "epoch": 0.05939377389404697, |
| "grad_norm": 0.06290453672409058, |
| "kl": 0.007605677310493775, |
| "learning_rate": 2.973684210526316e-06, |
| "loss": -0.0002, |
| "step": 435 |
| }, |
| { |
| "clip_ratio": 0.0010234627989120781, |
| "epoch": 0.05953031130529765, |
| "grad_norm": 0.062571220099926, |
| "kl": 0.007858202865463682, |
| "learning_rate": 2.9684210526315795e-06, |
| "loss": -0.0003, |
| "step": 436 |
| }, |
| { |
| "clip_ratio": 0.0007991675884113647, |
| "completion_length": 144.78125, |
| "epoch": 0.059666848716548336, |
| "grad_norm": 0.05652600899338722, |
| "kl": 0.00760183293823502, |
| "learning_rate": 2.9631578947368422e-06, |
| "loss": 0.0006, |
| "num_tokens": 1326470.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 437 |
| }, |
| { |
| "clip_ratio": 0.0007073700544424355, |
| "epoch": 0.059803386127799014, |
| "grad_norm": 0.05640224367380142, |
| "kl": 0.007448100674082525, |
| "learning_rate": 2.957894736842106e-06, |
| "loss": 0.0001, |
| "step": 438 |
| }, |
| { |
| "clip_ratio": 0.0007873274444136769, |
| "epoch": 0.0599399235390497, |
| "grad_norm": 0.05569564551115036, |
| "kl": 0.0073175277684640605, |
| "learning_rate": 2.9526315789473685e-06, |
| "loss": 0.0001, |
| "step": 439 |
| }, |
| { |
| "clip_ratio": 0.0002891639669542201, |
| "epoch": 0.060076460950300385, |
| "grad_norm": 0.05564608797430992, |
| "kl": 0.007433600101649063, |
| "learning_rate": 2.9473684210526317e-06, |
| "loss": -0.0001, |
| "step": 440 |
| }, |
| { |
| "clip_ratio": 0.00040302579873241484, |
| "completion_length": 120.625, |
| "epoch": 0.060212998361551064, |
| "grad_norm": 0.043795112520456314, |
| "kl": 0.00845221035706345, |
| "learning_rate": 2.942105263157895e-06, |
| "loss": -0.0, |
| "num_tokens": 1337326.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 441 |
| }, |
| { |
| "clip_ratio": 0.0004887530085397884, |
| "epoch": 0.06034953577280175, |
| "grad_norm": 0.04387158527970314, |
| "kl": 0.00806999751512194, |
| "learning_rate": 2.936842105263158e-06, |
| "loss": 0.0001, |
| "step": 442 |
| }, |
| { |
| "clip_ratio": 0.0009821199055295438, |
| "epoch": 0.06048607318405243, |
| "grad_norm": 0.043887924402952194, |
| "kl": 0.008568216500862036, |
| "learning_rate": 2.931578947368421e-06, |
| "loss": -0.0, |
| "step": 443 |
| }, |
| { |
| "clip_ratio": 0.00021701389050576836, |
| "epoch": 0.06062261059530311, |
| "grad_norm": 0.04335944354534149, |
| "kl": 0.008516778485500254, |
| "learning_rate": 2.9263157894736844e-06, |
| "loss": -0.0003, |
| "step": 444 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 126.6875, |
| "epoch": 0.0607591480065538, |
| "grad_norm": 0.03973577544093132, |
| "kl": 0.009528565780783538, |
| "learning_rate": 2.921052631578948e-06, |
| "loss": 0.0002, |
| "num_tokens": 1348776.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 445 |
| }, |
| { |
| "clip_ratio": 0.0005212095275055617, |
| "epoch": 0.060895685417804477, |
| "grad_norm": 0.04035574942827225, |
| "kl": 0.009289589856052771, |
| "learning_rate": 2.9157894736842107e-06, |
| "loss": 0.0001, |
| "step": 446 |
| }, |
| { |
| "clip_ratio": 0.0007640809926670045, |
| "epoch": 0.06103222282905516, |
| "grad_norm": 0.03905649483203888, |
| "kl": 0.009416846907697618, |
| "learning_rate": 2.9105263157894743e-06, |
| "loss": 0.0002, |
| "step": 447 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06116876024030585, |
| "grad_norm": 0.03937271237373352, |
| "kl": 0.009228991439158563, |
| "learning_rate": 2.905263157894737e-06, |
| "loss": 0.0001, |
| "step": 448 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 144.03125, |
| "epoch": 0.061305297651556526, |
| "grad_norm": 0.023931652307510376, |
| "kl": 0.006969279693294084, |
| "learning_rate": 2.9e-06, |
| "loss": 0.0, |
| "num_tokens": 1360501.0, |
| "reward": 0.859375, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.859375, |
| "step": 449 |
| }, |
| { |
| "clip_ratio": 0.00017959770048037171, |
| "epoch": 0.06144183506280721, |
| "grad_norm": 0.024019664153456688, |
| "kl": 0.0072047357316478156, |
| "learning_rate": 2.8947368421052634e-06, |
| "loss": 0.0, |
| "step": 450 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06157837247405789, |
| "grad_norm": 0.024044739082455635, |
| "kl": 0.0072458274189557415, |
| "learning_rate": 2.8894736842105265e-06, |
| "loss": -0.0, |
| "step": 451 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.061714909885308575, |
| "grad_norm": 0.02365250326693058, |
| "kl": 0.007240568047564011, |
| "learning_rate": 2.88421052631579e-06, |
| "loss": 0.0, |
| "step": 452 |
| }, |
| { |
| "clip_ratio": 0.0009471115772612393, |
| "completion_length": 155.625, |
| "epoch": 0.06185144729655926, |
| "grad_norm": 0.06520352512598038, |
| "kl": 0.008073884710029233, |
| "learning_rate": 2.878947368421053e-06, |
| "loss": -0.0001, |
| "num_tokens": 1372793.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.31605762243270874, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 453 |
| }, |
| { |
| "clip_ratio": 0.0010080113279400393, |
| "epoch": 0.06198798470780994, |
| "grad_norm": 0.06548015773296356, |
| "kl": 0.008144522056682035, |
| "learning_rate": 2.8736842105263164e-06, |
| "loss": -0.0, |
| "step": 454 |
| }, |
| { |
| "clip_ratio": 0.0003958662273362279, |
| "epoch": 0.062124522119060624, |
| "grad_norm": 0.06568895280361176, |
| "kl": 0.008249530437751673, |
| "learning_rate": 2.868421052631579e-06, |
| "loss": -0.0002, |
| "step": 455 |
| }, |
| { |
| "clip_ratio": 0.0010577178909443319, |
| "epoch": 0.0622610595303113, |
| "grad_norm": 0.06486720591783524, |
| "kl": 0.00807830964185996, |
| "learning_rate": 2.8631578947368423e-06, |
| "loss": -0.0002, |
| "step": 456 |
| }, |
| { |
| "clip_ratio": 0.00039798454963602126, |
| "completion_length": 138.46875, |
| "epoch": 0.06239759694156199, |
| "grad_norm": 0.05580438673496246, |
| "kl": 0.007516152116295416, |
| "learning_rate": 2.8578947368421055e-06, |
| "loss": -0.0002, |
| "num_tokens": 1384236.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.578125, |
| "step": 457 |
| }, |
| { |
| "clip_ratio": 0.00019654087373055518, |
| "epoch": 0.06253413435281267, |
| "grad_norm": 0.05768116936087608, |
| "kl": 0.00729399445845047, |
| "learning_rate": 2.8526315789473687e-06, |
| "loss": -0.0001, |
| "step": 458 |
| }, |
| { |
| "clip_ratio": 0.00018825300503522158, |
| "epoch": 0.06267067176406335, |
| "grad_norm": 0.05602006986737251, |
| "kl": 0.007249914233398158, |
| "learning_rate": 2.847368421052632e-06, |
| "loss": -0.0004, |
| "step": 459 |
| }, |
| { |
| "clip_ratio": 0.00038479387876577675, |
| "epoch": 0.06280720917531403, |
| "grad_norm": 0.05459784343838692, |
| "kl": 0.0073278494528494775, |
| "learning_rate": 2.842105263157895e-06, |
| "loss": -0.0004, |
| "step": 460 |
| }, |
| { |
| "clip_ratio": 0.001322940515819937, |
| "completion_length": 126.96875, |
| "epoch": 0.06294374658656472, |
| "grad_norm": 0.04787022992968559, |
| "kl": 0.007472993682313245, |
| "learning_rate": 2.8368421052631586e-06, |
| "loss": 0.0, |
| "num_tokens": 1395835.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 461 |
| }, |
| { |
| "clip_ratio": 0.0006167762912809849, |
| "epoch": 0.0630802839978154, |
| "grad_norm": 0.04787183925509453, |
| "kl": 0.007538449593994301, |
| "learning_rate": 2.8315789473684213e-06, |
| "loss": 0.0001, |
| "step": 462 |
| }, |
| { |
| "clip_ratio": 0.0002741228090599179, |
| "epoch": 0.06321682140906608, |
| "grad_norm": 0.04783927649259567, |
| "kl": 0.007550138026999775, |
| "learning_rate": 2.826315789473684e-06, |
| "loss": 0.0, |
| "step": 463 |
| }, |
| { |
| "clip_ratio": 0.001337673849775456, |
| "epoch": 0.06335335882031677, |
| "grad_norm": 0.04312347620725632, |
| "kl": 0.007741852998151444, |
| "learning_rate": 2.8210526315789476e-06, |
| "loss": -0.0002, |
| "step": 464 |
| }, |
| { |
| "clip_ratio": 0.00029588626784970984, |
| "completion_length": 134.53125, |
| "epoch": 0.06348989623156745, |
| "grad_norm": 0.05371793359518051, |
| "kl": 0.008338020328665152, |
| "learning_rate": 2.815789473684211e-06, |
| "loss": 0.0001, |
| "num_tokens": 1407072.0, |
| "reward": 0.828125, |
| "reward_std": 0.24865379929542542, |
| "rewards/reward_fn": 0.828125, |
| "step": 465 |
| }, |
| { |
| "clip_ratio": 9.55657524173148e-05, |
| "epoch": 0.06362643364281813, |
| "grad_norm": 0.052854083478450775, |
| "kl": 0.008835916931275278, |
| "learning_rate": 2.810526315789474e-06, |
| "loss": 0.0, |
| "step": 466 |
| }, |
| { |
| "clip_ratio": 9.55657524173148e-05, |
| "epoch": 0.06376297105406882, |
| "grad_norm": 0.05398806557059288, |
| "kl": 0.008691802664543502, |
| "learning_rate": 2.805263157894737e-06, |
| "loss": 0.0003, |
| "step": 467 |
| }, |
| { |
| "clip_ratio": 9.55657524173148e-05, |
| "epoch": 0.0638995084653195, |
| "grad_norm": 0.050633691251277924, |
| "kl": 0.008962789579527453, |
| "learning_rate": 2.8000000000000003e-06, |
| "loss": -0.0003, |
| "step": 468 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 155.9375, |
| "epoch": 0.06403604587657018, |
| "grad_norm": 0.054204683750867844, |
| "kl": 0.007828808353224304, |
| "learning_rate": 2.7947368421052635e-06, |
| "loss": -0.0001, |
| "num_tokens": 1419514.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 469 |
| }, |
| { |
| "clip_ratio": 0.0007495762692997232, |
| "epoch": 0.06417258328782087, |
| "grad_norm": 0.05434466153383255, |
| "kl": 0.007988040815689601, |
| "learning_rate": 2.789473684210526e-06, |
| "loss": 0.0, |
| "step": 470 |
| }, |
| { |
| "clip_ratio": 0.000399762429879047, |
| "epoch": 0.06430912069907155, |
| "grad_norm": 0.05508078634738922, |
| "kl": 0.008221916577895172, |
| "learning_rate": 2.7842105263157898e-06, |
| "loss": -0.0002, |
| "step": 471 |
| }, |
| { |
| "clip_ratio": 0.00021701389050576836, |
| "epoch": 0.06444565811032223, |
| "grad_norm": 0.05410420894622803, |
| "kl": 0.00835999096307205, |
| "learning_rate": 2.7789473684210525e-06, |
| "loss": -0.0004, |
| "step": 472 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.65625, |
| "epoch": 0.0645821955215729, |
| "grad_norm": 0.02417965605854988, |
| "kl": 0.00920310116634937, |
| "learning_rate": 2.773684210526316e-06, |
| "loss": 0.0001, |
| "num_tokens": 1430475.0, |
| "reward": 0.9156249761581421, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.9156249761581421, |
| "step": 473 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0647187329328236, |
| "grad_norm": 0.024912603199481964, |
| "kl": 0.009417042863788083, |
| "learning_rate": 2.7684210526315793e-06, |
| "loss": 0.0001, |
| "step": 474 |
| }, |
| { |
| "clip_ratio": 0.0004006410308647901, |
| "epoch": 0.06485527034407428, |
| "grad_norm": 0.02478739432990551, |
| "kl": 0.009008636669022962, |
| "learning_rate": 2.7631578947368424e-06, |
| "loss": -0.0, |
| "step": 475 |
| }, |
| { |
| "clip_ratio": 0.00020032051543239504, |
| "epoch": 0.06499180775532495, |
| "grad_norm": 0.024642430245876312, |
| "kl": 0.00952191120450152, |
| "learning_rate": 2.7578947368421056e-06, |
| "loss": 0.0, |
| "step": 476 |
| }, |
| { |
| "clip_ratio": 0.00049227150157094, |
| "completion_length": 139.46875, |
| "epoch": 0.06512834516657565, |
| "grad_norm": 0.0237932987511158, |
| "kl": 0.007173134734330233, |
| "learning_rate": 2.7526315789473683e-06, |
| "loss": 0.0001, |
| "num_tokens": 1442702.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 477 |
| }, |
| { |
| "clip_ratio": 0.0008282930066343397, |
| "epoch": 0.06526488257782632, |
| "grad_norm": 0.023669112473726273, |
| "kl": 0.007206353351648431, |
| "learning_rate": 2.747368421052632e-06, |
| "loss": 0.0001, |
| "step": 478 |
| }, |
| { |
| "clip_ratio": 0.00015624999650754035, |
| "epoch": 0.065401419989077, |
| "grad_norm": 0.023794015869498253, |
| "kl": 0.007167281808506232, |
| "learning_rate": 2.7421052631578947e-06, |
| "loss": 0.0, |
| "step": 479 |
| }, |
| { |
| "clip_ratio": 0.00015624999650754035, |
| "epoch": 0.0655379574003277, |
| "grad_norm": 0.02396044135093689, |
| "kl": 0.0072074738127412274, |
| "learning_rate": 2.7368421052631583e-06, |
| "loss": -0.0001, |
| "step": 480 |
| }, |
| { |
| "clip_ratio": 0.0010420576145406812, |
| "completion_length": 126.59375, |
| "epoch": 0.06567449481157837, |
| "grad_norm": 0.05958811938762665, |
| "kl": 0.010296369131538086, |
| "learning_rate": 2.7315789473684214e-06, |
| "loss": 0.0002, |
| "num_tokens": 1454273.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.578125, |
| "step": 481 |
| }, |
| { |
| "clip_ratio": 0.0004636886005755514, |
| "epoch": 0.06581103222282905, |
| "grad_norm": 0.05972179397940636, |
| "kl": 0.010362934131990187, |
| "learning_rate": 2.7263157894736846e-06, |
| "loss": -0.0, |
| "step": 482 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "epoch": 0.06594756963407974, |
| "grad_norm": 0.0592544786632061, |
| "kl": 0.01011465305055026, |
| "learning_rate": 2.7210526315789478e-06, |
| "loss": -0.0001, |
| "step": 483 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "epoch": 0.06608410704533042, |
| "grad_norm": 0.0606064610183239, |
| "kl": 0.010066984410514124, |
| "learning_rate": 2.7157894736842105e-06, |
| "loss": -0.0004, |
| "step": 484 |
| }, |
| { |
| "clip_ratio": 0.0007799234299454838, |
| "completion_length": 129.75, |
| "epoch": 0.0662206444565811, |
| "grad_norm": 0.08105815201997757, |
| "kl": 0.010437390141305514, |
| "learning_rate": 2.710526315789474e-06, |
| "loss": 0.0001, |
| "num_tokens": 1465153.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.37230759859085083, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 485 |
| }, |
| { |
| "clip_ratio": 0.0006660997751168907, |
| "epoch": 0.06635718186783178, |
| "grad_norm": 0.08110716938972473, |
| "kl": 0.010064481575682294, |
| "learning_rate": 2.705263157894737e-06, |
| "loss": 0.0001, |
| "step": 486 |
| }, |
| { |
| "clip_ratio": 0.0008750965353101492, |
| "epoch": 0.06649371927908247, |
| "grad_norm": 0.08124295622110367, |
| "kl": 0.010350861870392691, |
| "learning_rate": 2.7000000000000004e-06, |
| "loss": -0.0006, |
| "step": 487 |
| }, |
| { |
| "clip_ratio": 0.0006936128629604355, |
| "epoch": 0.06663025669033315, |
| "grad_norm": 0.0819404274225235, |
| "kl": 0.010304041315976065, |
| "learning_rate": 2.694736842105263e-06, |
| "loss": -0.0007, |
| "step": 488 |
| }, |
| { |
| "clip_ratio": 0.0010393823176855221, |
| "completion_length": 138.53125, |
| "epoch": 0.06676679410158383, |
| "grad_norm": 0.04559588059782982, |
| "kl": 0.009519384773739148, |
| "learning_rate": 2.6894736842105267e-06, |
| "loss": 0.0002, |
| "num_tokens": 1476366.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 489 |
| }, |
| { |
| "clip_ratio": 0.0010393823176855221, |
| "epoch": 0.06690333151283452, |
| "grad_norm": 0.04479571804404259, |
| "kl": 0.009794550598599017, |
| "learning_rate": 2.68421052631579e-06, |
| "loss": 0.0003, |
| "step": 490 |
| }, |
| { |
| "clip_ratio": 0.0010393823176855221, |
| "epoch": 0.0670398689240852, |
| "grad_norm": 0.044694747775793076, |
| "kl": 0.009446559291973244, |
| "learning_rate": 2.6789473684210526e-06, |
| "loss": 0.0001, |
| "step": 491 |
| }, |
| { |
| "clip_ratio": 0.0006281981040956452, |
| "epoch": 0.06717640633533588, |
| "grad_norm": 0.04453105479478836, |
| "kl": 0.009437322907615453, |
| "learning_rate": 2.6736842105263162e-06, |
| "loss": -0.0001, |
| "step": 492 |
| }, |
| { |
| "clip_ratio": 0.0004374073032522574, |
| "completion_length": 150.125, |
| "epoch": 0.06731294374658657, |
| "grad_norm": 0.0537416972219944, |
| "kl": 0.007753859412332531, |
| "learning_rate": 2.668421052631579e-06, |
| "loss": 0.0, |
| "num_tokens": 1488586.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.34620189666748047, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 493 |
| }, |
| { |
| "clip_ratio": 0.00046718136582057923, |
| "epoch": 0.06744948115783725, |
| "grad_norm": 0.05424033850431442, |
| "kl": 0.008118594545521773, |
| "learning_rate": 2.6631578947368426e-06, |
| "loss": 0.0001, |
| "step": 494 |
| }, |
| { |
| "clip_ratio": 0.0006564403010997921, |
| "epoch": 0.06758601856908793, |
| "grad_norm": 0.05376908928155899, |
| "kl": 0.007811405062966514, |
| "learning_rate": 2.6578947368421053e-06, |
| "loss": -0.0004, |
| "step": 495 |
| }, |
| { |
| "clip_ratio": 0.000517412438057363, |
| "epoch": 0.06772255598033862, |
| "grad_norm": 0.05373242124915123, |
| "kl": 0.007797598766046576, |
| "learning_rate": 2.652631578947369e-06, |
| "loss": -0.0002, |
| "step": 496 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 137.34375, |
| "epoch": 0.0678590933915893, |
| "grad_norm": 0.02003801055252552, |
| "kl": 0.008343649431481026, |
| "learning_rate": 2.6473684210526316e-06, |
| "loss": 0.0001, |
| "num_tokens": 1499993.0, |
| "reward": 0.859375, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.859375, |
| "step": 497 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06799563080283998, |
| "grad_norm": 0.019703811034560204, |
| "kl": 0.008276053864392452, |
| "learning_rate": 2.6421052631578948e-06, |
| "loss": 0.0, |
| "step": 498 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06813216821409065, |
| "grad_norm": 0.019753389060497284, |
| "kl": 0.008117678102280479, |
| "learning_rate": 2.6368421052631584e-06, |
| "loss": -0.0, |
| "step": 499 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06826870562534135, |
| "grad_norm": 0.019724242389202118, |
| "kl": 0.008083865948719904, |
| "learning_rate": 2.631578947368421e-06, |
| "loss": -0.0, |
| "step": 500 |
| }, |
| { |
| "clip_ratio": 0.0005328302504494786, |
| "completion_length": 147.03125, |
| "epoch": 0.06840524303659203, |
| "grad_norm": 0.04219144955277443, |
| "kl": 0.007861140082241036, |
| "learning_rate": 2.6263157894736847e-06, |
| "loss": 0.0, |
| "num_tokens": 1511322.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 501 |
| }, |
| { |
| "clip_ratio": 0.000689865424646996, |
| "epoch": 0.0685417804478427, |
| "grad_norm": 0.04225167632102966, |
| "kl": 0.007937694361316971, |
| "learning_rate": 2.6210526315789474e-06, |
| "loss": 0.0002, |
| "step": 502 |
| }, |
| { |
| "clip_ratio": 0.0010375593992648646, |
| "epoch": 0.0686783178590934, |
| "grad_norm": 0.04208582267165184, |
| "kl": 0.00758769222011324, |
| "learning_rate": 2.615789473684211e-06, |
| "loss": 0.0002, |
| "step": 503 |
| }, |
| { |
| "clip_ratio": 0.00039928323531057686, |
| "epoch": 0.06881485527034407, |
| "grad_norm": 0.04193231835961342, |
| "kl": 0.007824085136235226, |
| "learning_rate": 2.6105263157894738e-06, |
| "loss": -0.0, |
| "step": 504 |
| }, |
| { |
| "clip_ratio": 0.00011488970631035045, |
| "completion_length": 136.0625, |
| "epoch": 0.06895139268159475, |
| "grad_norm": 0.046760737895965576, |
| "kl": 0.007290848505363101, |
| "learning_rate": 2.605263157894737e-06, |
| "loss": 0.0002, |
| "num_tokens": 1522188.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 505 |
| }, |
| { |
| "clip_ratio": 0.0002297794126207009, |
| "epoch": 0.06908793009284545, |
| "grad_norm": 0.04790591448545456, |
| "kl": 0.007269349465786945, |
| "learning_rate": 2.6e-06, |
| "loss": 0.0001, |
| "step": 506 |
| }, |
| { |
| "clip_ratio": 0.00011488970631035045, |
| "epoch": 0.06922446750409612, |
| "grad_norm": 0.04738159105181694, |
| "kl": 0.007421690534101799, |
| "learning_rate": 2.5947368421052633e-06, |
| "loss": 0.0, |
| "step": 507 |
| }, |
| { |
| "clip_ratio": 0.0001849112450145185, |
| "epoch": 0.0693610049153468, |
| "grad_norm": 0.04717235639691353, |
| "kl": 0.00728582630836172, |
| "learning_rate": 2.589473684210527e-06, |
| "loss": -0.0001, |
| "step": 508 |
| }, |
| { |
| "clip_ratio": 0.0004856254963669926, |
| "completion_length": 126.90625, |
| "epoch": 0.0694975423265975, |
| "grad_norm": 0.05138390511274338, |
| "kl": 0.007572208542114822, |
| "learning_rate": 2.5842105263157896e-06, |
| "loss": 0.0001, |
| "num_tokens": 1533105.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 509 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.06963407973784817, |
| "grad_norm": 0.051094233989715576, |
| "kl": 0.007171828212449327, |
| "learning_rate": 2.578947368421053e-06, |
| "loss": -0.0001, |
| "step": 510 |
| }, |
| { |
| "clip_ratio": 0.0005696550651919097, |
| "epoch": 0.06977061714909885, |
| "grad_norm": 0.05045778304338455, |
| "kl": 0.007254811302118469, |
| "learning_rate": 2.573684210526316e-06, |
| "loss": -0.0002, |
| "step": 511 |
| }, |
| { |
| "clip_ratio": 0.00027243590739089996, |
| "epoch": 0.06990715456034953, |
| "grad_norm": 0.050060734152793884, |
| "kl": 0.0073423770954832435, |
| "learning_rate": 2.568421052631579e-06, |
| "loss": -0.0002, |
| "step": 512 |
| }, |
| { |
| "clip_ratio": 0.0009451095029362477, |
| "completion_length": 141.125, |
| "epoch": 0.07004369197160022, |
| "grad_norm": 0.04451264813542366, |
| "kl": 0.006996985550358659, |
| "learning_rate": 2.5631578947368422e-06, |
| "loss": 0.0001, |
| "num_tokens": 1545169.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 513 |
| }, |
| { |
| "clip_ratio": 0.000987816420092713, |
| "epoch": 0.0701802293828509, |
| "grad_norm": 0.04418221488595009, |
| "kl": 0.007054033278109273, |
| "learning_rate": 2.5578947368421054e-06, |
| "loss": 0.0003, |
| "step": 514 |
| }, |
| { |
| "clip_ratio": 0.0004747869024868123, |
| "epoch": 0.07031676679410158, |
| "grad_norm": 0.0442209355533123, |
| "kl": 0.006971489721763646, |
| "learning_rate": 2.552631578947369e-06, |
| "loss": 0.0001, |
| "step": 515 |
| }, |
| { |
| "clip_ratio": 0.0009743622722453438, |
| "epoch": 0.07045330420535227, |
| "grad_norm": 0.04460246115922928, |
| "kl": 0.006732409972755704, |
| "learning_rate": 2.5473684210526317e-06, |
| "loss": -0.0001, |
| "step": 516 |
| }, |
| { |
| "clip_ratio": 0.00052497441356536, |
| "completion_length": 174.5, |
| "epoch": 0.07058984161660295, |
| "grad_norm": 0.04996655508875847, |
| "kl": 0.007745866361801745, |
| "learning_rate": 2.5421052631578953e-06, |
| "loss": 0.0001, |
| "num_tokens": 1558285.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 517 |
| }, |
| { |
| "clip_ratio": 0.00032946215651463717, |
| "epoch": 0.07072637902785363, |
| "grad_norm": 0.05018087103962898, |
| "kl": 0.007652532476640772, |
| "learning_rate": 2.536842105263158e-06, |
| "loss": 0.0001, |
| "step": 518 |
| }, |
| { |
| "clip_ratio": 0.00046358232793863863, |
| "epoch": 0.07086291643910432, |
| "grad_norm": 0.050602760165929794, |
| "kl": 0.007677754623728106, |
| "learning_rate": 2.5315789473684212e-06, |
| "loss": 0.0002, |
| "step": 519 |
| }, |
| { |
| "clip_ratio": 0.0009925093909259886, |
| "epoch": 0.070999453850355, |
| "grad_norm": 0.05035364627838135, |
| "kl": 0.007573130151286023, |
| "learning_rate": 2.5263157894736844e-06, |
| "loss": 0.0, |
| "step": 520 |
| }, |
| { |
| "clip_ratio": 0.0007816088909748942, |
| "completion_length": 144.875, |
| "epoch": 0.07113599126160568, |
| "grad_norm": 0.06629616022109985, |
| "kl": 0.00734264800848905, |
| "learning_rate": 2.5210526315789475e-06, |
| "loss": -0.0002, |
| "num_tokens": 1569389.0, |
| "reward": 0.6343750357627869, |
| "reward_std": 0.36360570788383484, |
| "rewards/reward_fn": 0.6343750357627869, |
| "step": 521 |
| }, |
| { |
| "clip_ratio": 0.00032899269717745483, |
| "epoch": 0.07127252867285637, |
| "grad_norm": 0.06583209335803986, |
| "kl": 0.007242169034725521, |
| "learning_rate": 2.5157894736842107e-06, |
| "loss": -0.0003, |
| "step": 522 |
| }, |
| { |
| "clip_ratio": 0.0012217525363666937, |
| "epoch": 0.07140906608410705, |
| "grad_norm": 0.06630554050207138, |
| "kl": 0.007015092756773811, |
| "learning_rate": 2.510526315789474e-06, |
| "loss": -0.0002, |
| "step": 523 |
| }, |
| { |
| "clip_ratio": 0.0007908968691481277, |
| "epoch": 0.07154560349535773, |
| "grad_norm": 0.06656049937009811, |
| "kl": 0.00725256945588626, |
| "learning_rate": 2.5052631578947375e-06, |
| "loss": -0.0005, |
| "step": 524 |
| }, |
| { |
| "clip_ratio": 0.0003033053653780371, |
| "completion_length": 148.75, |
| "epoch": 0.0716821409066084, |
| "grad_norm": 0.037184614688158035, |
| "kl": 0.006560517471370986, |
| "learning_rate": 2.5e-06, |
| "loss": 0.0, |
| "num_tokens": 1581053.0, |
| "reward": 0.578125, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.578125, |
| "step": 525 |
| }, |
| { |
| "clip_ratio": 0.00015862943837419152, |
| "epoch": 0.0718186783178591, |
| "grad_norm": 0.03728421404957771, |
| "kl": 0.006451733977883123, |
| "learning_rate": 2.4947368421052634e-06, |
| "loss": -0.0001, |
| "step": 526 |
| }, |
| { |
| "clip_ratio": 0.0003033053653780371, |
| "epoch": 0.07195521572910978, |
| "grad_norm": 0.03703222796320915, |
| "kl": 0.006404903520888183, |
| "learning_rate": 2.4894736842105265e-06, |
| "loss": -0.0001, |
| "step": 527 |
| }, |
| { |
| "clip_ratio": 0.00015862943837419152, |
| "epoch": 0.07209175314036045, |
| "grad_norm": 0.037218909710645676, |
| "kl": 0.006483051227405667, |
| "learning_rate": 2.4842105263157897e-06, |
| "loss": -0.0002, |
| "step": 528 |
| }, |
| { |
| "clip_ratio": 0.0006236314366105944, |
| "completion_length": 147.46875, |
| "epoch": 0.07222829055161115, |
| "grad_norm": 0.05063549429178238, |
| "kl": 0.006116248892794829, |
| "learning_rate": 2.478947368421053e-06, |
| "loss": 0.0, |
| "num_tokens": 1593232.0, |
| "reward": 0.71875, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.71875, |
| "step": 529 |
| }, |
| { |
| "clip_ratio": 0.00015862943837419152, |
| "epoch": 0.07236482796286182, |
| "grad_norm": 0.049550946801900864, |
| "kl": 0.006080309889512137, |
| "learning_rate": 2.473684210526316e-06, |
| "loss": -0.0002, |
| "step": 530 |
| }, |
| { |
| "clip_ratio": 0.0006244407704798505, |
| "epoch": 0.0725013653741125, |
| "grad_norm": 0.051187340170145035, |
| "kl": 0.006060313564375974, |
| "learning_rate": 2.468421052631579e-06, |
| "loss": 0.0, |
| "step": 531 |
| }, |
| { |
| "clip_ratio": 0.00018382353300694376, |
| "epoch": 0.0726379027853632, |
| "grad_norm": 0.05119786411523819, |
| "kl": 0.006236782101041172, |
| "learning_rate": 2.4631578947368424e-06, |
| "loss": -0.0002, |
| "step": 532 |
| }, |
| { |
| "clip_ratio": 0.0012781979457940906, |
| "completion_length": 143.15625, |
| "epoch": 0.07277444019661387, |
| "grad_norm": 0.05053151398897171, |
| "kl": 0.007381562500086147, |
| "learning_rate": 2.4578947368421055e-06, |
| "loss": 0.0004, |
| "num_tokens": 1605429.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.578125, |
| "step": 533 |
| }, |
| { |
| "clip_ratio": 0.00042551118531264365, |
| "epoch": 0.07291097760786455, |
| "grad_norm": 0.05032571405172348, |
| "kl": 0.0075953817868139595, |
| "learning_rate": 2.4526315789473687e-06, |
| "loss": 0.0004, |
| "step": 534 |
| }, |
| { |
| "clip_ratio": 0.0012935490376548842, |
| "epoch": 0.07304751501911524, |
| "grad_norm": 0.04841490462422371, |
| "kl": 0.007569821740617044, |
| "learning_rate": 2.447368421052632e-06, |
| "loss": 0.0003, |
| "step": 535 |
| }, |
| { |
| "clip_ratio": 0.0005802662926726043, |
| "epoch": 0.07318405243036592, |
| "grad_norm": 0.04850556701421738, |
| "kl": 0.007605544735270087, |
| "learning_rate": 2.442105263157895e-06, |
| "loss": 0.0, |
| "step": 536 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 151.71875, |
| "epoch": 0.0733205898416166, |
| "grad_norm": 0.04175799712538719, |
| "kl": 0.007070479194226209, |
| "learning_rate": 2.436842105263158e-06, |
| "loss": 0.0001, |
| "num_tokens": 1618260.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 537 |
| }, |
| { |
| "clip_ratio": 0.0002460629912093282, |
| "epoch": 0.07345712725286728, |
| "grad_norm": 0.04488982632756233, |
| "kl": 0.007328775136556942, |
| "learning_rate": 2.4315789473684213e-06, |
| "loss": 0.0001, |
| "step": 538 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07359366466411797, |
| "grad_norm": 0.043785080313682556, |
| "kl": 0.007057562266709283, |
| "learning_rate": 2.4263157894736845e-06, |
| "loss": 0.0001, |
| "step": 539 |
| }, |
| { |
| "clip_ratio": 0.00010279605339746922, |
| "epoch": 0.07373020207536865, |
| "grad_norm": 0.044333186000585556, |
| "kl": 0.007193806872237474, |
| "learning_rate": 2.4210526315789477e-06, |
| "loss": 0.0001, |
| "step": 540 |
| }, |
| { |
| "clip_ratio": 0.0011420008086133748, |
| "completion_length": 162.46875, |
| "epoch": 0.07386673948661933, |
| "grad_norm": 0.06641445308923721, |
| "kl": 0.00672327966458397, |
| "learning_rate": 2.415789473684211e-06, |
| "loss": 0.0002, |
| "num_tokens": 1631035.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.3636057376861572, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 541 |
| }, |
| { |
| "clip_ratio": 0.0007206795853562653, |
| "epoch": 0.07400327689787002, |
| "grad_norm": 0.06626348942518234, |
| "kl": 0.006686398162855767, |
| "learning_rate": 2.410526315789474e-06, |
| "loss": 0.0003, |
| "step": 542 |
| }, |
| { |
| "clip_ratio": 0.0014276769070420414, |
| "epoch": 0.0741398143091207, |
| "grad_norm": 0.06716456264257431, |
| "kl": 0.006765739897673484, |
| "learning_rate": 2.4052631578947367e-06, |
| "loss": -0.0001, |
| "step": 543 |
| }, |
| { |
| "clip_ratio": 0.0015944738406687975, |
| "epoch": 0.07427635172037138, |
| "grad_norm": 0.06432369351387024, |
| "kl": 0.006832511709944811, |
| "learning_rate": 2.4000000000000003e-06, |
| "loss": -0.0003, |
| "step": 544 |
| }, |
| { |
| "clip_ratio": 0.0004705899045802653, |
| "completion_length": 122.15625, |
| "epoch": 0.07441288913162207, |
| "grad_norm": 0.04856768250465393, |
| "kl": 0.007571559981442988, |
| "learning_rate": 2.3947368421052635e-06, |
| "loss": 0.0, |
| "num_tokens": 1642104.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 545 |
| }, |
| { |
| "clip_ratio": 0.000202922077733092, |
| "epoch": 0.07454942654287275, |
| "grad_norm": 0.04934373125433922, |
| "kl": 0.007731453602900729, |
| "learning_rate": 2.3894736842105266e-06, |
| "loss": -0.0, |
| "step": 546 |
| }, |
| { |
| "clip_ratio": 0.00015394088404718786, |
| "epoch": 0.07468596395412343, |
| "grad_norm": 0.04486023634672165, |
| "kl": 0.008048816678638104, |
| "learning_rate": 2.38421052631579e-06, |
| "loss": -0.0001, |
| "step": 547 |
| }, |
| { |
| "clip_ratio": 0.00022644927958026528, |
| "epoch": 0.07482250136537412, |
| "grad_norm": 0.041965968906879425, |
| "kl": 0.007910460844868794, |
| "learning_rate": 2.378947368421053e-06, |
| "loss": -0.0002, |
| "step": 548 |
| }, |
| { |
| "clip_ratio": 0.00038580247201025486, |
| "completion_length": 135.21875, |
| "epoch": 0.0749590387766248, |
| "grad_norm": 0.06485315412282944, |
| "kl": 0.006781648771720938, |
| "learning_rate": 2.373684210526316e-06, |
| "loss": 0.0, |
| "num_tokens": 1653419.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 549 |
| }, |
| { |
| "clip_ratio": 0.0007230819610413164, |
| "epoch": 0.07509557618787548, |
| "grad_norm": 0.06703505665063858, |
| "kl": 0.007002713697147556, |
| "learning_rate": 2.368421052631579e-06, |
| "loss": -0.0001, |
| "step": 550 |
| }, |
| { |
| "clip_ratio": 0.00019654087373055518, |
| "epoch": 0.07523211359912615, |
| "grad_norm": 0.06371887028217316, |
| "kl": 0.007053271110635251, |
| "learning_rate": 2.363157894736842e-06, |
| "loss": -0.0001, |
| "step": 551 |
| }, |
| { |
| "clip_ratio": 0.0006285735289566219, |
| "epoch": 0.07536865101037685, |
| "grad_norm": 0.06411947309970856, |
| "kl": 0.007059614843456075, |
| "learning_rate": 2.357894736842105e-06, |
| "loss": -0.0004, |
| "step": 552 |
| }, |
| { |
| "clip_ratio": 0.00035511364694684744, |
| "completion_length": 118.03125, |
| "epoch": 0.07550518842162753, |
| "grad_norm": 0.04954632744193077, |
| "kl": 0.007099473055859562, |
| "learning_rate": 2.352631578947369e-06, |
| "loss": 0.0001, |
| "num_tokens": 1664648.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 553 |
| }, |
| { |
| "clip_ratio": 0.00035511364694684744, |
| "epoch": 0.0756417258328782, |
| "grad_norm": 0.045967843383550644, |
| "kl": 0.0074233199848094955, |
| "learning_rate": 2.347368421052632e-06, |
| "loss": 0.0001, |
| "step": 554 |
| }, |
| { |
| "clip_ratio": 0.00035511364694684744, |
| "epoch": 0.0757782632441289, |
| "grad_norm": 0.04629616066813469, |
| "kl": 0.00743957648955984, |
| "learning_rate": 2.342105263157895e-06, |
| "loss": -0.0001, |
| "step": 555 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07591480065537957, |
| "grad_norm": 0.04565320163965225, |
| "kl": 0.007361977390246466, |
| "learning_rate": 2.3368421052631583e-06, |
| "loss": -0.0001, |
| "step": 556 |
| }, |
| { |
| "clip_ratio": 0.0010432856870465912, |
| "completion_length": 163.4375, |
| "epoch": 0.07605133806663025, |
| "grad_norm": 0.051344793289899826, |
| "kl": 0.00732030239305459, |
| "learning_rate": 2.331578947368421e-06, |
| "loss": 0.0002, |
| "num_tokens": 1677582.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 557 |
| }, |
| { |
| "clip_ratio": 0.00034594954195199534, |
| "epoch": 0.07618787547788095, |
| "grad_norm": 0.05109322816133499, |
| "kl": 0.007265964050020557, |
| "learning_rate": 2.326315789473684e-06, |
| "loss": 0.0, |
| "step": 558 |
| }, |
| { |
| "clip_ratio": 0.0005207576905377209, |
| "epoch": 0.07632441288913162, |
| "grad_norm": 0.04683186486363411, |
| "kl": 0.007142915565054864, |
| "learning_rate": 2.3210526315789473e-06, |
| "loss": -0.0001, |
| "step": 559 |
| }, |
| { |
| "clip_ratio": 0.00012860081915277988, |
| "epoch": 0.0764609503003823, |
| "grad_norm": 0.05104099586606026, |
| "kl": 0.007422650160151534, |
| "learning_rate": 2.3157894736842105e-06, |
| "loss": -0.0001, |
| "step": 560 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 132.65625, |
| "epoch": 0.07659748771163298, |
| "grad_norm": 0.04083140566945076, |
| "kl": 0.006990984169533476, |
| "learning_rate": 2.310526315789474e-06, |
| "loss": 0.0002, |
| "num_tokens": 1688787.0, |
| "reward": 0.831250011920929, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.831250011920929, |
| "step": 561 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07673402512288367, |
| "grad_norm": 0.04104629158973694, |
| "kl": 0.007018320131464861, |
| "learning_rate": 2.3052631578947373e-06, |
| "loss": 0.0002, |
| "step": 562 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07687056253413435, |
| "grad_norm": 0.040892329066991806, |
| "kl": 0.007225662586279213, |
| "learning_rate": 2.3000000000000004e-06, |
| "loss": 0.0001, |
| "step": 563 |
| }, |
| { |
| "clip_ratio": 0.00018825300503522158, |
| "epoch": 0.07700709994538503, |
| "grad_norm": 0.03983571380376816, |
| "kl": 0.007063861732603982, |
| "learning_rate": 2.294736842105263e-06, |
| "loss": 0.0001, |
| "step": 564 |
| }, |
| { |
| "clip_ratio": 0.0003124999930150807, |
| "completion_length": 119.9375, |
| "epoch": 0.07714363735663572, |
| "grad_norm": 0.06113090366125107, |
| "kl": 0.007402963827189524, |
| "learning_rate": 2.2894736842105263e-06, |
| "loss": -0.0001, |
| "num_tokens": 1699409.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 565 |
| }, |
| { |
| "clip_ratio": 0.00020424836839083582, |
| "epoch": 0.0772801747678864, |
| "grad_norm": 0.060313157737255096, |
| "kl": 0.00762724902597256, |
| "learning_rate": 2.2842105263157895e-06, |
| "loss": -0.0001, |
| "step": 566 |
| }, |
| { |
| "clip_ratio": 0.00012254902685526758, |
| "epoch": 0.07741671217913708, |
| "grad_norm": 0.060309067368507385, |
| "kl": 0.0074914450669893995, |
| "learning_rate": 2.2789473684210527e-06, |
| "loss": -0.0002, |
| "step": 567 |
| }, |
| { |
| "clip_ratio": 0.0005357142799766734, |
| "epoch": 0.07755324959038777, |
| "grad_norm": 0.05862335488200188, |
| "kl": 0.007220967221655883, |
| "learning_rate": 2.273684210526316e-06, |
| "loss": -0.0004, |
| "step": 568 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 143.09375, |
| "epoch": 0.07768978700163845, |
| "grad_norm": 0.06805431842803955, |
| "kl": 0.00756039610132575, |
| "learning_rate": 2.268421052631579e-06, |
| "loss": -0.0002, |
| "num_tokens": 1711644.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 569 |
| }, |
| { |
| "clip_ratio": 0.0005361070216167718, |
| "epoch": 0.07782632441288913, |
| "grad_norm": 0.06884022057056427, |
| "kl": 0.007600243574415799, |
| "learning_rate": 2.2631578947368426e-06, |
| "loss": 0.0002, |
| "step": 570 |
| }, |
| { |
| "clip_ratio": 0.0008021659596124664, |
| "epoch": 0.07796286182413982, |
| "grad_norm": 0.06727788597345352, |
| "kl": 0.007387209720036481, |
| "learning_rate": 2.2578947368421057e-06, |
| "loss": 0.0002, |
| "step": 571 |
| }, |
| { |
| "clip_ratio": 0.0007823739288141951, |
| "epoch": 0.0780993992353905, |
| "grad_norm": 0.06563364714384079, |
| "kl": 0.007573030743515119, |
| "learning_rate": 2.2526315789473685e-06, |
| "loss": -0.0003, |
| "step": 572 |
| }, |
| { |
| "clip_ratio": 0.0010938818741124123, |
| "completion_length": 122.3125, |
| "epoch": 0.07823593664664118, |
| "grad_norm": 0.044389545917510986, |
| "kl": 0.009747960364620667, |
| "learning_rate": 2.2473684210526316e-06, |
| "loss": 0.0003, |
| "num_tokens": 1722642.0, |
| "reward": 0.859375, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.859375, |
| "step": 573 |
| }, |
| { |
| "clip_ratio": 0.0005661231989506632, |
| "epoch": 0.07837247405789186, |
| "grad_norm": 0.0443134680390358, |
| "kl": 0.009794213394343387, |
| "learning_rate": 2.242105263157895e-06, |
| "loss": 0.0002, |
| "step": 574 |
| }, |
| { |
| "clip_ratio": 0.0011706486984621733, |
| "epoch": 0.07850901146914255, |
| "grad_norm": 0.04379402846097946, |
| "kl": 0.009853087249211967, |
| "learning_rate": 2.236842105263158e-06, |
| "loss": 0.0002, |
| "step": 575 |
| }, |
| { |
| "clip_ratio": 0.0006424369057640433, |
| "epoch": 0.07864554888039323, |
| "grad_norm": 0.043600961565971375, |
| "kl": 0.00974890669749584, |
| "learning_rate": 2.231578947368421e-06, |
| "loss": 0.0001, |
| "step": 576 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 131.25, |
| "epoch": 0.0787820862916439, |
| "grad_norm": 0.03951617702841759, |
| "kl": 0.006896804377902299, |
| "learning_rate": 2.2263157894736843e-06, |
| "loss": 0.0001, |
| "num_tokens": 1734490.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 577 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0789186237028946, |
| "grad_norm": 0.03939719498157501, |
| "kl": 0.006927944006747566, |
| "learning_rate": 2.221052631578948e-06, |
| "loss": -0.0001, |
| "step": 578 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07905516111414528, |
| "grad_norm": 0.03932217136025429, |
| "kl": 0.006925714696990326, |
| "learning_rate": 2.2157894736842106e-06, |
| "loss": -0.0, |
| "step": 579 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07919169852539595, |
| "grad_norm": 0.03965163603425026, |
| "kl": 0.0069147960530244745, |
| "learning_rate": 2.2105263157894738e-06, |
| "loss": -0.0002, |
| "step": 580 |
| }, |
| { |
| "clip_ratio": 0.00020833333837799728, |
| "completion_length": 150.375, |
| "epoch": 0.07932823593664665, |
| "grad_norm": 0.028819842264056206, |
| "kl": 0.0078005624454817735, |
| "learning_rate": 2.205263157894737e-06, |
| "loss": 0.0002, |
| "num_tokens": 1747254.0, |
| "reward": 0.690625011920929, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.690625011920929, |
| "step": 581 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.07946477334789732, |
| "grad_norm": 0.0284147709608078, |
| "kl": 0.008087361544312444, |
| "learning_rate": 2.2e-06, |
| "loss": 0.0001, |
| "step": 582 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.079601310759148, |
| "grad_norm": 0.02833622694015503, |
| "kl": 0.0077161458611954, |
| "learning_rate": 2.1947368421052633e-06, |
| "loss": 0.0002, |
| "step": 583 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0797378481703987, |
| "grad_norm": 0.02837838977575302, |
| "kl": 0.00781893861858407, |
| "learning_rate": 2.1894736842105264e-06, |
| "loss": -0.0, |
| "step": 584 |
| }, |
| { |
| "clip_ratio": 0.0002541951907915063, |
| "completion_length": 146.59375, |
| "epoch": 0.07987438558164937, |
| "grad_norm": 0.051355116069316864, |
| "kl": 0.008217885988415219, |
| "learning_rate": 2.1842105263157896e-06, |
| "loss": -0.0, |
| "num_tokens": 1758973.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 585 |
| }, |
| { |
| "clip_ratio": 0.0001911315048346296, |
| "epoch": 0.08001092299290005, |
| "grad_norm": 0.05148646607995033, |
| "kl": 0.008319878026668448, |
| "learning_rate": 2.1789473684210528e-06, |
| "loss": 0.0002, |
| "step": 586 |
| }, |
| { |
| "clip_ratio": 0.0004453266737982631, |
| "epoch": 0.08014746040415073, |
| "grad_norm": 0.05023907124996185, |
| "kl": 0.008321528570377268, |
| "learning_rate": 2.173684210526316e-06, |
| "loss": -0.0001, |
| "step": 587 |
| }, |
| { |
| "clip_ratio": 0.00020258629956515506, |
| "epoch": 0.08028399781540142, |
| "grad_norm": 0.05123789235949516, |
| "kl": 0.008500380994519219, |
| "learning_rate": 2.168421052631579e-06, |
| "loss": -0.0003, |
| "step": 588 |
| }, |
| { |
| "clip_ratio": 0.000541363813681528, |
| "completion_length": 129.1875, |
| "epoch": 0.0804205352266521, |
| "grad_norm": 0.052199020981788635, |
| "kl": 0.006949372938834131, |
| "learning_rate": 2.1631578947368423e-06, |
| "loss": 0.0001, |
| "num_tokens": 1770835.0, |
| "reward": 0.71875, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.71875, |
| "step": 589 |
| }, |
| { |
| "clip_ratio": 0.0012156298689660616, |
| "epoch": 0.08055707263790278, |
| "grad_norm": 0.051836010068655014, |
| "kl": 0.007309889231692068, |
| "learning_rate": 2.1578947368421054e-06, |
| "loss": 0.0001, |
| "step": 590 |
| }, |
| { |
| "clip_ratio": 0.000952955218963325, |
| "epoch": 0.08069361004915347, |
| "grad_norm": 0.05232127383351326, |
| "kl": 0.007312422480026726, |
| "learning_rate": 2.1526315789473686e-06, |
| "loss": 0.0003, |
| "step": 591 |
| }, |
| { |
| "clip_ratio": 0.000949845023569651, |
| "epoch": 0.08083014746040415, |
| "grad_norm": 0.05187045410275459, |
| "kl": 0.007411205020616762, |
| "learning_rate": 2.1473684210526317e-06, |
| "loss": -0.0, |
| "step": 592 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 122.28125, |
| "epoch": 0.08096668487165483, |
| "grad_norm": 0.05353539064526558, |
| "kl": 0.00940216681919992, |
| "learning_rate": 2.142105263157895e-06, |
| "loss": -0.0, |
| "num_tokens": 1782120.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 593 |
| }, |
| { |
| "clip_ratio": 0.0007851513801142573, |
| "epoch": 0.08110322228290552, |
| "grad_norm": 0.0503874309360981, |
| "kl": 0.009512974858807866, |
| "learning_rate": 2.136842105263158e-06, |
| "loss": -0.0001, |
| "step": 594 |
| }, |
| { |
| "clip_ratio": 0.00021701389050576836, |
| "epoch": 0.0812397596941562, |
| "grad_norm": 0.05228208750486374, |
| "kl": 0.009625030914321542, |
| "learning_rate": 2.1315789473684212e-06, |
| "loss": -0.0002, |
| "step": 595 |
| }, |
| { |
| "clip_ratio": 0.00036036250821780413, |
| "epoch": 0.08137629710540688, |
| "grad_norm": 0.051866378635168076, |
| "kl": 0.009751744168170262, |
| "learning_rate": 2.1263157894736844e-06, |
| "loss": -0.0002, |
| "step": 596 |
| }, |
| { |
| "clip_ratio": 0.0013829681847710162, |
| "completion_length": 127.59375, |
| "epoch": 0.08151283451665757, |
| "grad_norm": 0.08412730693817139, |
| "kl": 0.008330287142598536, |
| "learning_rate": 2.1210526315789476e-06, |
| "loss": 0.0006, |
| "num_tokens": 1794151.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 597 |
| }, |
| { |
| "clip_ratio": 0.001611070372746326, |
| "epoch": 0.08164937192790825, |
| "grad_norm": 0.07318681478500366, |
| "kl": 0.008474633730656933, |
| "learning_rate": 2.1157894736842107e-06, |
| "loss": 0.0004, |
| "step": 598 |
| }, |
| { |
| "clip_ratio": 0.0006414203671738505, |
| "epoch": 0.08178590933915893, |
| "grad_norm": 0.07287193834781647, |
| "kl": 0.008253247106040362, |
| "learning_rate": 2.110526315789474e-06, |
| "loss": -0.0001, |
| "step": 599 |
| }, |
| { |
| "clip_ratio": 0.0005382472154451534, |
| "epoch": 0.0819224467504096, |
| "grad_norm": 0.07386298477649689, |
| "kl": 0.008286618642159738, |
| "learning_rate": 2.105263157894737e-06, |
| "loss": -0.0003, |
| "step": 600 |
| }, |
| { |
| "clip_ratio": 0.0012213949084980413, |
| "completion_length": 138.84375, |
| "epoch": 0.0820589841616603, |
| "grad_norm": 0.05285279080271721, |
| "kl": 0.009743083974171896, |
| "learning_rate": 2.1000000000000002e-06, |
| "loss": 0.0002, |
| "num_tokens": 1805866.0, |
| "reward": 0.831250011920929, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.831250011920929, |
| "step": 601 |
| }, |
| { |
| "clip_ratio": 0.000794481486082077, |
| "epoch": 0.08219552157291098, |
| "grad_norm": 0.05252443999052048, |
| "kl": 0.009864158193522599, |
| "learning_rate": 2.0947368421052634e-06, |
| "loss": -0.0, |
| "step": 602 |
| }, |
| { |
| "clip_ratio": 0.00046173963346518576, |
| "epoch": 0.08233205898416165, |
| "grad_norm": 0.05354541167616844, |
| "kl": 0.0097685690925573, |
| "learning_rate": 2.0894736842105266e-06, |
| "loss": -0.0, |
| "step": 603 |
| }, |
| { |
| "clip_ratio": 0.0013237248931545764, |
| "epoch": 0.08246859639541235, |
| "grad_norm": 0.053757648915052414, |
| "kl": 0.009642576609621756, |
| "learning_rate": 2.0842105263157897e-06, |
| "loss": -0.0, |
| "step": 604 |
| }, |
| { |
| "clip_ratio": 0.0009758359956322238, |
| "completion_length": 124.0625, |
| "epoch": 0.08260513380666303, |
| "grad_norm": 0.06367083638906479, |
| "kl": 0.007945968201966025, |
| "learning_rate": 2.078947368421053e-06, |
| "loss": -0.0, |
| "num_tokens": 1816340.0, |
| "reward": 0.578125, |
| "reward_std": 0.2511056959629059, |
| "rewards/reward_fn": 0.578125, |
| "step": 605 |
| }, |
| { |
| "clip_ratio": 0.0004987843421986327, |
| "epoch": 0.0827416712179137, |
| "grad_norm": 0.05984797328710556, |
| "kl": 0.007906621969596017, |
| "learning_rate": 2.073684210526316e-06, |
| "loss": -0.0, |
| "step": 606 |
| }, |
| { |
| "clip_ratio": 0.0007197852100944147, |
| "epoch": 0.0828782086291644, |
| "grad_norm": 0.0647590309381485, |
| "kl": 0.007820824612281285, |
| "learning_rate": 2.068421052631579e-06, |
| "loss": -0.0001, |
| "step": 607 |
| }, |
| { |
| "clip_ratio": 0.00016983695968519896, |
| "epoch": 0.08301474604041507, |
| "grad_norm": 0.06323892623186111, |
| "kl": 0.007831803883163957, |
| "learning_rate": 2.0631578947368424e-06, |
| "loss": -0.0002, |
| "step": 608 |
| }, |
| { |
| "clip_ratio": 0.0004282477020751685, |
| "completion_length": 153.0625, |
| "epoch": 0.08315128345166575, |
| "grad_norm": 0.043793871998786926, |
| "kl": 0.007419599896820728, |
| "learning_rate": 2.0578947368421055e-06, |
| "loss": 0.0001, |
| "num_tokens": 1828706.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 609 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08328782086291645, |
| "grad_norm": 0.04405396804213524, |
| "kl": 0.0071023885102476925, |
| "learning_rate": 2.0526315789473687e-06, |
| "loss": 0.0001, |
| "step": 610 |
| }, |
| { |
| "clip_ratio": 0.0001516990305390209, |
| "epoch": 0.08342435827416712, |
| "grad_norm": 0.04358115419745445, |
| "kl": 0.007294165137864184, |
| "learning_rate": 2.047368421052632e-06, |
| "loss": 0.0001, |
| "step": 611 |
| }, |
| { |
| "clip_ratio": 0.00026260505546815693, |
| "epoch": 0.0835608956854178, |
| "grad_norm": 0.04348994418978691, |
| "kl": 0.007233174270368181, |
| "learning_rate": 2.042105263157895e-06, |
| "loss": 0.0001, |
| "step": 612 |
| }, |
| { |
| "clip_ratio": 0.0004900417698081583, |
| "completion_length": 133.375, |
| "epoch": 0.08369743309666848, |
| "grad_norm": 0.04587121307849884, |
| "kl": 0.007063197328534443, |
| "learning_rate": 2.036842105263158e-06, |
| "loss": 0.0001, |
| "num_tokens": 1839574.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 613 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08383397050791917, |
| "grad_norm": 0.046325068920850754, |
| "kl": 0.007178902902523987, |
| "learning_rate": 2.031578947368421e-06, |
| "loss": 0.0002, |
| "step": 614 |
| }, |
| { |
| "clip_ratio": 0.00033457671815995127, |
| "epoch": 0.08397050791916985, |
| "grad_norm": 0.04607764631509781, |
| "kl": 0.007151073587010615, |
| "learning_rate": 2.026315789473684e-06, |
| "loss": -0.0001, |
| "step": 615 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "epoch": 0.08410704533042053, |
| "grad_norm": 0.046034447848796844, |
| "kl": 0.0071356187836499885, |
| "learning_rate": 2.0210526315789477e-06, |
| "loss": -0.0, |
| "step": 616 |
| }, |
| { |
| "clip_ratio": 0.000898550933925435, |
| "completion_length": 159.46875, |
| "epoch": 0.08424358274167122, |
| "grad_norm": 0.04303830489516258, |
| "kl": 0.007923301353002898, |
| "learning_rate": 2.015789473684211e-06, |
| "loss": 0.0002, |
| "num_tokens": 1852633.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 617 |
| }, |
| { |
| "clip_ratio": 0.0006053575198166072, |
| "epoch": 0.0843801201529219, |
| "grad_norm": 0.04329138994216919, |
| "kl": 0.00798736748402007, |
| "learning_rate": 2.010526315789474e-06, |
| "loss": -0.0001, |
| "step": 618 |
| }, |
| { |
| "clip_ratio": 0.00032021683000493795, |
| "epoch": 0.08451665756417258, |
| "grad_norm": 0.043333351612091064, |
| "kl": 0.008088444446912035, |
| "learning_rate": 2.005263157894737e-06, |
| "loss": -0.0, |
| "step": 619 |
| }, |
| { |
| "clip_ratio": 0.00047568946320097893, |
| "epoch": 0.08465319497542327, |
| "grad_norm": 0.042955655604600906, |
| "kl": 0.008101311686914414, |
| "learning_rate": 2.0000000000000003e-06, |
| "loss": -0.0002, |
| "step": 620 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 121.8125, |
| "epoch": 0.08478973238667395, |
| "grad_norm": 0.044429562985897064, |
| "kl": 0.008902769950509537, |
| "learning_rate": 1.994736842105263e-06, |
| "loss": 0.0002, |
| "num_tokens": 1864083.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 621 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08492626979792463, |
| "grad_norm": 0.04492293670773506, |
| "kl": 0.008974922697234433, |
| "learning_rate": 1.9894736842105262e-06, |
| "loss": 0.0001, |
| "step": 622 |
| }, |
| { |
| "clip_ratio": 0.00024224806111305952, |
| "epoch": 0.08506280720917532, |
| "grad_norm": 0.044010475277900696, |
| "kl": 0.008985094667878002, |
| "learning_rate": 1.9842105263157894e-06, |
| "loss": -0.0001, |
| "step": 623 |
| }, |
| { |
| "clip_ratio": 0.00028935185400769114, |
| "epoch": 0.085199344620426, |
| "grad_norm": 0.0438762903213501, |
| "kl": 0.008959770326327998, |
| "learning_rate": 1.978947368421053e-06, |
| "loss": 0.0001, |
| "step": 624 |
| }, |
| { |
| "clip_ratio": 0.0003730945463757962, |
| "completion_length": 122.09375, |
| "epoch": 0.08533588203167668, |
| "grad_norm": 0.04697028920054436, |
| "kl": 0.00760080355394166, |
| "learning_rate": 1.973684210526316e-06, |
| "loss": 0.0, |
| "num_tokens": 1875630.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 625 |
| }, |
| { |
| "clip_ratio": 0.0003730945463757962, |
| "epoch": 0.08547241944292736, |
| "grad_norm": 0.04671761393547058, |
| "kl": 0.007557677134172991, |
| "learning_rate": 1.9684210526315793e-06, |
| "loss": -0.0003, |
| "step": 626 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08560895685417805, |
| "grad_norm": 0.04660893604159355, |
| "kl": 0.0074983131780754775, |
| "learning_rate": 1.9631578947368425e-06, |
| "loss": -0.0002, |
| "step": 627 |
| }, |
| { |
| "clip_ratio": 0.00017655367264524102, |
| "epoch": 0.08574549426542873, |
| "grad_norm": 0.04735223948955536, |
| "kl": 0.007747615985863376, |
| "learning_rate": 1.9578947368421052e-06, |
| "loss": -0.0005, |
| "step": 628 |
| }, |
| { |
| "clip_ratio": 0.0012378239625832066, |
| "completion_length": 116.84375, |
| "epoch": 0.0858820316766794, |
| "grad_norm": 0.06293011456727982, |
| "kl": 0.010361742941313423, |
| "learning_rate": 1.9526315789473684e-06, |
| "loss": 0.0003, |
| "num_tokens": 1886917.0, |
| "reward": 0.71875, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.71875, |
| "step": 629 |
| }, |
| { |
| "clip_ratio": 0.0002604166802484542, |
| "epoch": 0.0860185690879301, |
| "grad_norm": 0.0631277933716774, |
| "kl": 0.010464194914675318, |
| "learning_rate": 1.9473684210526315e-06, |
| "loss": 0.0003, |
| "step": 630 |
| }, |
| { |
| "clip_ratio": 0.001014503781334497, |
| "epoch": 0.08615510649918078, |
| "grad_norm": 0.06210329383611679, |
| "kl": 0.010841057155630551, |
| "learning_rate": 1.9421052631578947e-06, |
| "loss": 0.0003, |
| "step": 631 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08629164391043145, |
| "grad_norm": 0.0609845370054245, |
| "kl": 0.010285505442880094, |
| "learning_rate": 1.936842105263158e-06, |
| "loss": -0.0003, |
| "step": 632 |
| }, |
| { |
| "clip_ratio": 0.0012845874298363924, |
| "completion_length": 116.53125, |
| "epoch": 0.08642818132168215, |
| "grad_norm": 0.08124419301748276, |
| "kl": 0.010507312086701859, |
| "learning_rate": 1.9315789473684215e-06, |
| "loss": 0.0001, |
| "num_tokens": 1897946.0, |
| "reward": 0.71875, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.71875, |
| "step": 633 |
| }, |
| { |
| "clip_ratio": 0.0016739532729843631, |
| "epoch": 0.08656471873293282, |
| "grad_norm": 0.08041945844888687, |
| "kl": 0.010619292144838255, |
| "learning_rate": 1.9263157894736846e-06, |
| "loss": 0.0, |
| "step": 634 |
| }, |
| { |
| "clip_ratio": 0.0009326871659141034, |
| "epoch": 0.0867012561441835, |
| "grad_norm": 0.08038292080163956, |
| "kl": 0.01045711140614003, |
| "learning_rate": 1.9210526315789474e-06, |
| "loss": -0.0002, |
| "step": 635 |
| }, |
| { |
| "clip_ratio": 0.0010508133709663525, |
| "epoch": 0.0868377935554342, |
| "grad_norm": 0.07905282080173492, |
| "kl": 0.010489227279322222, |
| "learning_rate": 1.9157894736842105e-06, |
| "loss": -0.0003, |
| "step": 636 |
| }, |
| { |
| "clip_ratio": 0.00017361111531499773, |
| "completion_length": 131.59375, |
| "epoch": 0.08697433096668487, |
| "grad_norm": 0.05166640132665634, |
| "kl": 0.010781421136925928, |
| "learning_rate": 1.9105263157894737e-06, |
| "loss": 0.0001, |
| "num_tokens": 1909637.0, |
| "reward": 0.6312500238418579, |
| "reward_std": 0.17113018035888672, |
| "rewards/reward_fn": 0.6312500238418579, |
| "step": 637 |
| }, |
| { |
| "clip_ratio": 0.00022163119865581393, |
| "epoch": 0.08711086837793555, |
| "grad_norm": 0.06559828668832779, |
| "kl": 0.011350437969667837, |
| "learning_rate": 1.905263157894737e-06, |
| "loss": 0.0003, |
| "step": 638 |
| }, |
| { |
| "clip_ratio": 0.00022163119865581393, |
| "epoch": 0.08724740578918623, |
| "grad_norm": 0.05125059559941292, |
| "kl": 0.010893267026403919, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": 0.0001, |
| "step": 639 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08738394320043692, |
| "grad_norm": 0.05015575885772705, |
| "kl": 0.010615745253744535, |
| "learning_rate": 1.8947368421052634e-06, |
| "loss": 0.0, |
| "step": 640 |
| }, |
| { |
| "clip_ratio": 0.0003571428533177823, |
| "completion_length": 123.375, |
| "epoch": 0.0875204806116876, |
| "grad_norm": 0.061218321323394775, |
| "kl": 0.00817414597258903, |
| "learning_rate": 1.8894736842105266e-06, |
| "loss": 0.0003, |
| "num_tokens": 1920713.0, |
| "reward": 0.71875, |
| "reward_std": 0.289951890707016, |
| "rewards/reward_fn": 0.71875, |
| "step": 641 |
| }, |
| { |
| "clip_ratio": 0.0012050134828314185, |
| "epoch": 0.08765701802293828, |
| "grad_norm": 0.060746241360902786, |
| "kl": 0.008164621773175895, |
| "learning_rate": 1.8842105263157895e-06, |
| "loss": 0.0, |
| "step": 642 |
| }, |
| { |
| "clip_ratio": 0.0003124999930150807, |
| "epoch": 0.08779355543418897, |
| "grad_norm": 0.06054810434579849, |
| "kl": 0.0077883233316242695, |
| "learning_rate": 1.8789473684210527e-06, |
| "loss": -0.0001, |
| "step": 643 |
| }, |
| { |
| "clip_ratio": 0.00017361111531499773, |
| "epoch": 0.08793009284543965, |
| "grad_norm": 0.060438014566898346, |
| "kl": 0.008346897368028294, |
| "learning_rate": 1.8736842105263158e-06, |
| "loss": -0.0003, |
| "step": 644 |
| }, |
| { |
| "clip_ratio": 0.0003396739193703979, |
| "completion_length": 120.78125, |
| "epoch": 0.08806663025669033, |
| "grad_norm": 0.05400171875953674, |
| "kl": 0.009491481905570254, |
| "learning_rate": 1.868421052631579e-06, |
| "loss": 0.0001, |
| "num_tokens": 1931342.0, |
| "reward": 0.6906249523162842, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.6906249523162842, |
| "step": 645 |
| }, |
| { |
| "clip_ratio": 0.00014880952949170023, |
| "epoch": 0.08820316766794102, |
| "grad_norm": 0.055399730801582336, |
| "kl": 0.009387899342982564, |
| "learning_rate": 1.8631578947368424e-06, |
| "loss": -0.0, |
| "step": 646 |
| }, |
| { |
| "clip_ratio": 0.00014880952949170023, |
| "epoch": 0.0883397050791917, |
| "grad_norm": 0.05491754040122032, |
| "kl": 0.00944241025717929, |
| "learning_rate": 1.8578947368421055e-06, |
| "loss": 0.0, |
| "step": 647 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08847624249044238, |
| "grad_norm": 0.05160977318882942, |
| "kl": 0.00919472768146079, |
| "learning_rate": 1.8526315789473687e-06, |
| "loss": -0.0002, |
| "step": 648 |
| }, |
| { |
| "clip_ratio": 0.00034357255208306015, |
| "completion_length": 132.71875, |
| "epoch": 0.08861277990169307, |
| "grad_norm": 0.05538962781429291, |
| "kl": 0.010922728404693771, |
| "learning_rate": 1.8473684210526319e-06, |
| "loss": 0.0002, |
| "num_tokens": 1943053.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 649 |
| }, |
| { |
| "clip_ratio": 0.00068842270411551, |
| "epoch": 0.08874931731294375, |
| "grad_norm": 0.05578883737325668, |
| "kl": 0.011023718376236502, |
| "learning_rate": 1.8421052631578948e-06, |
| "loss": 0.0003, |
| "step": 650 |
| }, |
| { |
| "clip_ratio": 0.00017857142665889114, |
| "epoch": 0.08888585472419443, |
| "grad_norm": 0.05544004961848259, |
| "kl": 0.01113046848331578, |
| "learning_rate": 1.836842105263158e-06, |
| "loss": 0.0001, |
| "step": 651 |
| }, |
| { |
| "clip_ratio": 0.00016801075253169984, |
| "epoch": 0.0890223921354451, |
| "grad_norm": 0.05554305016994476, |
| "kl": 0.010989264948875643, |
| "learning_rate": 1.8315789473684211e-06, |
| "loss": 0.0001, |
| "step": 652 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 137.46875, |
| "epoch": 0.0891589295466958, |
| "grad_norm": 0.026127351447939873, |
| "kl": 0.009274293479393236, |
| "learning_rate": 1.8263157894736843e-06, |
| "loss": 0.0001, |
| "num_tokens": 1954796.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 653 |
| }, |
| { |
| "clip_ratio": 0.00021404109429568052, |
| "epoch": 0.08929546695794648, |
| "grad_norm": 0.026443179696798325, |
| "kl": 0.009282741550123319, |
| "learning_rate": 1.8210526315789475e-06, |
| "loss": 0.0001, |
| "step": 654 |
| }, |
| { |
| "clip_ratio": 0.00021404109429568052, |
| "epoch": 0.08943200436919715, |
| "grad_norm": 0.02507867105305195, |
| "kl": 0.00907987051323289, |
| "learning_rate": 1.8157894736842109e-06, |
| "loss": 0.0, |
| "step": 655 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.08956854178044785, |
| "grad_norm": 0.025512471795082092, |
| "kl": 0.009064778816537, |
| "learning_rate": 1.810526315789474e-06, |
| "loss": 0.0, |
| "step": 656 |
| }, |
| { |
| "clip_ratio": 0.00021853146608918905, |
| "completion_length": 114.40625, |
| "epoch": 0.08970507919169853, |
| "grad_norm": 0.039977312088012695, |
| "kl": 0.011268600435869303, |
| "learning_rate": 1.805263157894737e-06, |
| "loss": 0.0, |
| "num_tokens": 1965937.0, |
| "reward": 0.9437500238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.9437500238418579, |
| "step": 657 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0898416166029492, |
| "grad_norm": 0.039484865963459015, |
| "kl": 0.01169414146716008, |
| "learning_rate": 1.8000000000000001e-06, |
| "loss": 0.0, |
| "step": 658 |
| }, |
| { |
| "clip_ratio": 0.0008360648353118449, |
| "epoch": 0.0899781540141999, |
| "grad_norm": 0.039373256266117096, |
| "kl": 0.01142600730236154, |
| "learning_rate": 1.7947368421052633e-06, |
| "loss": 0.0, |
| "step": 659 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09011469142545057, |
| "grad_norm": 0.03879743441939354, |
| "kl": 0.011224691275856458, |
| "learning_rate": 1.7894736842105265e-06, |
| "loss": -0.0001, |
| "step": 660 |
| }, |
| { |
| "clip_ratio": 0.0006016710540279746, |
| "completion_length": 131.0625, |
| "epoch": 0.09025122883670125, |
| "grad_norm": 0.06215175986289978, |
| "kl": 0.009479137479502242, |
| "learning_rate": 1.7842105263157896e-06, |
| "loss": 0.0001, |
| "num_tokens": 1977315.0, |
| "reward": 0.831250011920929, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.831250011920929, |
| "step": 661 |
| }, |
| { |
| "clip_ratio": 0.001295591879170388, |
| "epoch": 0.09038776624795195, |
| "grad_norm": 0.06316965818405151, |
| "kl": 0.009921427787048742, |
| "learning_rate": 1.7789473684210528e-06, |
| "loss": 0.0002, |
| "step": 662 |
| }, |
| { |
| "clip_ratio": 0.0004600110769388266, |
| "epoch": 0.09052430365920262, |
| "grad_norm": 0.06075277924537659, |
| "kl": 0.009608675216441043, |
| "learning_rate": 1.7736842105263162e-06, |
| "loss": -0.0003, |
| "step": 663 |
| }, |
| { |
| "clip_ratio": 0.0007406523654935881, |
| "epoch": 0.0906608410704533, |
| "grad_norm": 0.06145086511969566, |
| "kl": 0.009493120276601985, |
| "learning_rate": 1.768421052631579e-06, |
| "loss": -0.0004, |
| "step": 664 |
| }, |
| { |
| "clip_ratio": 0.0011664281773846596, |
| "completion_length": 119.5, |
| "epoch": 0.09079737848170398, |
| "grad_norm": 0.0636189803481102, |
| "kl": 0.009904831276799086, |
| "learning_rate": 1.7631578947368423e-06, |
| "loss": 0.0002, |
| "num_tokens": 1988307.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 665 |
| }, |
| { |
| "clip_ratio": 0.0003094059356953949, |
| "epoch": 0.09093391589295467, |
| "grad_norm": 0.053713634610176086, |
| "kl": 0.01019222864852054, |
| "learning_rate": 1.7578947368421054e-06, |
| "loss": 0.0001, |
| "step": 666 |
| }, |
| { |
| "clip_ratio": 0.0008770763233769685, |
| "epoch": 0.09107045330420535, |
| "grad_norm": 0.06056439131498337, |
| "kl": 0.010176277442951687, |
| "learning_rate": 1.7526315789473686e-06, |
| "loss": 0.0003, |
| "step": 667 |
| }, |
| { |
| "clip_ratio": 0.0005676703876815736, |
| "epoch": 0.09120699071545603, |
| "grad_norm": 0.05292230099439621, |
| "kl": 0.010127040943189058, |
| "learning_rate": 1.7473684210526318e-06, |
| "loss": -0.0001, |
| "step": 668 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 152.4375, |
| "epoch": 0.09134352812670672, |
| "grad_norm": 0.059158798307180405, |
| "kl": 0.009955877947504632, |
| "learning_rate": 1.742105263157895e-06, |
| "loss": 0.0002, |
| "num_tokens": 2000637.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 669 |
| }, |
| { |
| "clip_ratio": 0.0008623772009741515, |
| "epoch": 0.0914800655379574, |
| "grad_norm": 0.060120780020952225, |
| "kl": 0.009407724930497352, |
| "learning_rate": 1.736842105263158e-06, |
| "loss": 0.0001, |
| "step": 670 |
| }, |
| { |
| "clip_ratio": 0.00014880952949170023, |
| "epoch": 0.09161660294920808, |
| "grad_norm": 0.062163278460502625, |
| "kl": 0.009934681169397663, |
| "learning_rate": 1.731578947368421e-06, |
| "loss": -0.0001, |
| "step": 671 |
| }, |
| { |
| "clip_ratio": 0.00027901786961592734, |
| "epoch": 0.09175314036045877, |
| "grad_norm": 0.05974072590470314, |
| "kl": 0.009933095832820982, |
| "learning_rate": 1.7263157894736842e-06, |
| "loss": 0.0, |
| "step": 672 |
| }, |
| { |
| "clip_ratio": 0.00014467592700384557, |
| "completion_length": 113.75, |
| "epoch": 0.09188967777170945, |
| "grad_norm": 0.06370711326599121, |
| "kl": 0.009484674970735796, |
| "learning_rate": 1.7210526315789474e-06, |
| "loss": 0.0002, |
| "num_tokens": 2011513.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.578125, |
| "step": 673 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09202621518296013, |
| "grad_norm": 0.06518139690160751, |
| "kl": 0.009496869577560574, |
| "learning_rate": 1.7157894736842107e-06, |
| "loss": 0.0001, |
| "step": 674 |
| }, |
| { |
| "clip_ratio": 0.00034340660204179585, |
| "epoch": 0.09216275259421082, |
| "grad_norm": 0.06475136429071426, |
| "kl": 0.009787194590899162, |
| "learning_rate": 1.710526315789474e-06, |
| "loss": -0.0001, |
| "step": 675 |
| }, |
| { |
| "clip_ratio": 0.00034340660204179585, |
| "epoch": 0.0922992900054615, |
| "grad_norm": 0.06383711099624634, |
| "kl": 0.009830301656620577, |
| "learning_rate": 1.705263157894737e-06, |
| "loss": -0.0001, |
| "step": 676 |
| }, |
| { |
| "clip_ratio": 0.000779913054429926, |
| "completion_length": 130.5625, |
| "epoch": 0.09243582741671218, |
| "grad_norm": 0.05045098438858986, |
| "kl": 0.007975322056154255, |
| "learning_rate": 1.7000000000000002e-06, |
| "loss": 0.0002, |
| "num_tokens": 2022787.0, |
| "reward": 0.8031249642372131, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.8031249642372131, |
| "step": 677 |
| }, |
| { |
| "clip_ratio": 0.0006429948698496446, |
| "epoch": 0.09257236482796286, |
| "grad_norm": 0.04944043233990669, |
| "kl": 0.007966513803694397, |
| "learning_rate": 1.6947368421052632e-06, |
| "loss": -0.0001, |
| "step": 678 |
| }, |
| { |
| "clip_ratio": 0.0007752536766929552, |
| "epoch": 0.09270890223921355, |
| "grad_norm": 0.05094648152589798, |
| "kl": 0.007958199421409518, |
| "learning_rate": 1.6894736842105264e-06, |
| "loss": 0.0001, |
| "step": 679 |
| }, |
| { |
| "clip_ratio": 0.00038959863741183653, |
| "epoch": 0.09284543965046423, |
| "grad_norm": 0.050485141575336456, |
| "kl": 0.007655591682123486, |
| "learning_rate": 1.6842105263157895e-06, |
| "loss": 0.0, |
| "step": 680 |
| }, |
| { |
| "clip_ratio": 0.0005374903994379565, |
| "completion_length": 115.9375, |
| "epoch": 0.0929819770617149, |
| "grad_norm": 0.03506218641996384, |
| "kl": 0.010533176384342369, |
| "learning_rate": 1.6789473684210527e-06, |
| "loss": 0.0003, |
| "num_tokens": 2033581.0, |
| "reward": 0.831250011920929, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.831250011920929, |
| "step": 681 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0931185144729656, |
| "grad_norm": 0.03745941072702408, |
| "kl": 0.010677461941668298, |
| "learning_rate": 1.673684210526316e-06, |
| "loss": -0.0, |
| "step": 682 |
| }, |
| { |
| "clip_ratio": 0.0001296680566156283, |
| "epoch": 0.09325505188421628, |
| "grad_norm": 0.037467096000909805, |
| "kl": 0.010990642382239457, |
| "learning_rate": 1.6684210526315792e-06, |
| "loss": 0.0, |
| "step": 683 |
| }, |
| { |
| "clip_ratio": 0.0003890041552949697, |
| "epoch": 0.09339158929546695, |
| "grad_norm": 0.03714219853281975, |
| "kl": 0.010533199238125235, |
| "learning_rate": 1.6631578947368424e-06, |
| "loss": 0.0001, |
| "step": 684 |
| }, |
| { |
| "clip_ratio": 0.0007841863262001425, |
| "completion_length": 135.15625, |
| "epoch": 0.09352812670671765, |
| "grad_norm": 0.058404963463544846, |
| "kl": 0.009558297533658333, |
| "learning_rate": 1.6578947368421053e-06, |
| "loss": -0.0, |
| "num_tokens": 2044498.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 685 |
| }, |
| { |
| "clip_ratio": 0.00028935185400769114, |
| "epoch": 0.09366466411796832, |
| "grad_norm": 0.058251362293958664, |
| "kl": 0.00946732052398147, |
| "learning_rate": 1.6526315789473685e-06, |
| "loss": -0.0001, |
| "step": 686 |
| }, |
| { |
| "clip_ratio": 0.0009979581664083526, |
| "epoch": 0.093801201529219, |
| "grad_norm": 0.0560067743062973, |
| "kl": 0.009381448006024584, |
| "learning_rate": 1.6473684210526317e-06, |
| "loss": -0.0002, |
| "step": 687 |
| }, |
| { |
| "clip_ratio": 0.0005787037080153823, |
| "epoch": 0.0939377389404697, |
| "grad_norm": 0.055572494864463806, |
| "kl": 0.00957166004809551, |
| "learning_rate": 1.6421052631578948e-06, |
| "loss": -0.0003, |
| "step": 688 |
| }, |
| { |
| "clip_ratio": 0.0006241734517971054, |
| "completion_length": 141.59375, |
| "epoch": 0.09407427635172037, |
| "grad_norm": 0.06296925991773605, |
| "kl": 0.009739323417306878, |
| "learning_rate": 1.636842105263158e-06, |
| "loss": -0.0002, |
| "num_tokens": 2056605.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 689 |
| }, |
| { |
| "clip_ratio": 0.0008112376381177455, |
| "epoch": 0.09421081376297105, |
| "grad_norm": 0.06260309368371964, |
| "kl": 0.009592891088686883, |
| "learning_rate": 1.6315789473684212e-06, |
| "loss": 0.0002, |
| "step": 690 |
| }, |
| { |
| "clip_ratio": 0.0015434112428920344, |
| "epoch": 0.09434735117422173, |
| "grad_norm": 0.06253664195537567, |
| "kl": 0.009875656258373056, |
| "learning_rate": 1.6263157894736845e-06, |
| "loss": 0.0001, |
| "step": 691 |
| }, |
| { |
| "clip_ratio": 0.0012417175312293693, |
| "epoch": 0.09448388858547242, |
| "grad_norm": 0.06115971505641937, |
| "kl": 0.009915292619552929, |
| "learning_rate": 1.6210526315789473e-06, |
| "loss": -0.0001, |
| "step": 692 |
| }, |
| { |
| "clip_ratio": 0.0006988612585701048, |
| "completion_length": 123.53125, |
| "epoch": 0.0946204259967231, |
| "grad_norm": 0.036156997084617615, |
| "kl": 0.009653015731601045, |
| "learning_rate": 1.6157894736842106e-06, |
| "loss": 0.0, |
| "num_tokens": 2068170.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 693 |
| }, |
| { |
| "clip_ratio": 0.0005157603009138256, |
| "epoch": 0.09475696340797378, |
| "grad_norm": 0.03611351177096367, |
| "kl": 0.009572211987688206, |
| "learning_rate": 1.6105263157894738e-06, |
| "loss": 0.0001, |
| "step": 694 |
| }, |
| { |
| "clip_ratio": 0.0002367424312978983, |
| "epoch": 0.09489350081922447, |
| "grad_norm": 0.03618909791111946, |
| "kl": 0.00937497743871063, |
| "learning_rate": 1.605263157894737e-06, |
| "loss": 0.0, |
| "step": 695 |
| }, |
| { |
| "clip_ratio": 0.00027173911803402007, |
| "epoch": 0.09503003823047515, |
| "grad_norm": 0.036196816712617874, |
| "kl": 0.009594166782335378, |
| "learning_rate": 1.6000000000000001e-06, |
| "loss": -0.0001, |
| "step": 696 |
| }, |
| { |
| "clip_ratio": 0.0002813780592987314, |
| "completion_length": 133.9375, |
| "epoch": 0.09516657564172583, |
| "grad_norm": 0.051047615706920624, |
| "kl": 0.010030516001279466, |
| "learning_rate": 1.5947368421052633e-06, |
| "loss": 0.0002, |
| "num_tokens": 2079616.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 697 |
| }, |
| { |
| "clip_ratio": 0.0002976257965201512, |
| "epoch": 0.09530311305297652, |
| "grad_norm": 0.05097904056310654, |
| "kl": 0.01036051158735063, |
| "learning_rate": 1.5894736842105265e-06, |
| "loss": -0.0002, |
| "step": 698 |
| }, |
| { |
| "clip_ratio": 0.00013185653369873762, |
| "epoch": 0.0954396504642272, |
| "grad_norm": 0.05117359384894371, |
| "kl": 0.010354827187256888, |
| "learning_rate": 1.5842105263157894e-06, |
| "loss": 0.0001, |
| "step": 699 |
| }, |
| { |
| "clip_ratio": 0.00014952152559999377, |
| "epoch": 0.09557618787547788, |
| "grad_norm": 0.05063074827194214, |
| "kl": 0.010228267652564682, |
| "learning_rate": 1.5789473684210526e-06, |
| "loss": -0.0003, |
| "step": 700 |
| }, |
| { |
| "clip_ratio": 0.0003962820628657937, |
| "completion_length": 178.96875, |
| "epoch": 0.09571272528672857, |
| "grad_norm": 0.06450162827968597, |
| "kl": 0.00951177971001016, |
| "learning_rate": 1.573684210526316e-06, |
| "loss": 0.0002, |
| "num_tokens": 2093331.0, |
| "reward": 0.4906250238418579, |
| "reward_std": 0.3587018847465515, |
| "rewards/reward_fn": 0.4906250238418579, |
| "step": 701 |
| }, |
| { |
| "clip_ratio": 0.0012379881809465587, |
| "epoch": 0.09584926269797925, |
| "grad_norm": 0.0656437948346138, |
| "kl": 0.009871184774965513, |
| "learning_rate": 1.5684210526315791e-06, |
| "loss": 0.0, |
| "step": 702 |
| }, |
| { |
| "clip_ratio": 0.0007504209643229842, |
| "epoch": 0.09598580010922993, |
| "grad_norm": 0.06479347497224808, |
| "kl": 0.009675588124082424, |
| "learning_rate": 1.5631578947368423e-06, |
| "loss": -0.0002, |
| "step": 703 |
| }, |
| { |
| "clip_ratio": 0.0013040981139056385, |
| "epoch": 0.0961223375204806, |
| "grad_norm": 0.061526622623205185, |
| "kl": 0.009392668194777798, |
| "learning_rate": 1.5578947368421054e-06, |
| "loss": -0.0003, |
| "step": 704 |
| }, |
| { |
| "clip_ratio": 0.0004366117645986378, |
| "completion_length": 119.78125, |
| "epoch": 0.0962588749317313, |
| "grad_norm": 0.04564754292368889, |
| "kl": 0.009969821738195606, |
| "learning_rate": 1.5526315789473686e-06, |
| "loss": 0.0003, |
| "num_tokens": 2104092.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.12990380823612213, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 705 |
| }, |
| { |
| "clip_ratio": 0.0004366117645986378, |
| "epoch": 0.09639541234298198, |
| "grad_norm": 0.045418836176395416, |
| "kl": 0.009776668084668927, |
| "learning_rate": 1.5473684210526316e-06, |
| "loss": 0.0002, |
| "step": 706 |
| }, |
| { |
| "clip_ratio": 0.00019054877338930964, |
| "epoch": 0.09653194975423265, |
| "grad_norm": 0.045270442962646484, |
| "kl": 0.00967571978981141, |
| "learning_rate": 1.5421052631578947e-06, |
| "loss": 0.0002, |
| "step": 707 |
| }, |
| { |
| "clip_ratio": 0.0003325942234368995, |
| "epoch": 0.09666848716548335, |
| "grad_norm": 0.0454433336853981, |
| "kl": 0.009787519709789194, |
| "learning_rate": 1.5368421052631579e-06, |
| "loss": 0.0001, |
| "step": 708 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 126.34375, |
| "epoch": 0.09680502457673403, |
| "grad_norm": 0.02350000850856304, |
| "kl": 0.00850833029835485, |
| "learning_rate": 1.5315789473684213e-06, |
| "loss": 0.0002, |
| "num_tokens": 2115007.0, |
| "reward": 0.71875, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.71875, |
| "step": 709 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0969415619879847, |
| "grad_norm": 0.023005494847893715, |
| "kl": 0.008384571425267495, |
| "learning_rate": 1.5263157894736844e-06, |
| "loss": 0.0, |
| "step": 710 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0970780993992354, |
| "grad_norm": 0.02326674945652485, |
| "kl": 0.008348509953066241, |
| "learning_rate": 1.5210526315789476e-06, |
| "loss": 0.0, |
| "step": 711 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09721463681048607, |
| "grad_norm": 0.023169759660959244, |
| "kl": 0.008243571959610563, |
| "learning_rate": 1.5157894736842108e-06, |
| "loss": 0.0, |
| "step": 712 |
| }, |
| { |
| "clip_ratio": 0.00046202958037611097, |
| "completion_length": 124.03125, |
| "epoch": 0.09735117422173675, |
| "grad_norm": 0.04860498756170273, |
| "kl": 0.009658148563175928, |
| "learning_rate": 1.5105263157894737e-06, |
| "loss": 0.0, |
| "num_tokens": 2126100.0, |
| "reward": 0.859375, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.859375, |
| "step": 713 |
| }, |
| { |
| "clip_ratio": 0.0009270593582186848, |
| "epoch": 0.09748771163298744, |
| "grad_norm": 0.048729058355093, |
| "kl": 0.009522449159703683, |
| "learning_rate": 1.5052631578947369e-06, |
| "loss": 0.0, |
| "step": 714 |
| }, |
| { |
| "clip_ratio": 0.001418196057784371, |
| "epoch": 0.09762424904423812, |
| "grad_norm": 0.048881612718105316, |
| "kl": 0.009417750574357342, |
| "learning_rate": 1.5e-06, |
| "loss": -0.0001, |
| "step": 715 |
| }, |
| { |
| "clip_ratio": 0.0004186267906334251, |
| "epoch": 0.0977607864554888, |
| "grad_norm": 0.048408348113298416, |
| "kl": 0.009555224532959983, |
| "learning_rate": 1.4947368421052632e-06, |
| "loss": -0.0001, |
| "step": 716 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.53125, |
| "epoch": 0.09789732386673948, |
| "grad_norm": 0.04021313041448593, |
| "kl": 0.011427787831053138, |
| "learning_rate": 1.4894736842105264e-06, |
| "loss": 0.0002, |
| "num_tokens": 2137421.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 717 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09803386127799017, |
| "grad_norm": 0.04058316349983215, |
| "kl": 0.011506663853651844, |
| "learning_rate": 1.4842105263157897e-06, |
| "loss": -0.0001, |
| "step": 718 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09817039868924085, |
| "grad_norm": 0.04035284370183945, |
| "kl": 0.011317572338157333, |
| "learning_rate": 1.478947368421053e-06, |
| "loss": 0.0001, |
| "step": 719 |
| }, |
| { |
| "clip_ratio": 0.00018382353300694376, |
| "epoch": 0.09830693610049153, |
| "grad_norm": 0.04018305987119675, |
| "kl": 0.011519249426783063, |
| "learning_rate": 1.4736842105263159e-06, |
| "loss": -0.0, |
| "step": 720 |
| }, |
| { |
| "clip_ratio": 0.0010667011374607682, |
| "completion_length": 120.625, |
| "epoch": 0.09844347351174222, |
| "grad_norm": 0.06840135902166367, |
| "kl": 0.010099568207806442, |
| "learning_rate": 1.468421052631579e-06, |
| "loss": 0.0002, |
| "num_tokens": 2148217.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.578125, |
| "step": 721 |
| }, |
| { |
| "clip_ratio": 0.0005217391299083829, |
| "epoch": 0.0985800109229929, |
| "grad_norm": 0.06865006685256958, |
| "kl": 0.010052972807898186, |
| "learning_rate": 1.4631578947368422e-06, |
| "loss": 0.0, |
| "step": 722 |
| }, |
| { |
| "clip_ratio": 0.0007070135907270014, |
| "epoch": 0.09871654833424358, |
| "grad_norm": 0.06821943819522858, |
| "kl": 0.010255348970531486, |
| "learning_rate": 1.4578947368421053e-06, |
| "loss": 0.0001, |
| "step": 723 |
| }, |
| { |
| "clip_ratio": 0.0010091461590491235, |
| "epoch": 0.09885308574549427, |
| "grad_norm": 0.06956622749567032, |
| "kl": 0.010160391895624343, |
| "learning_rate": 1.4526315789473685e-06, |
| "loss": -0.0002, |
| "step": 724 |
| }, |
| { |
| "clip_ratio": 0.0005287147941999137, |
| "completion_length": 132.0625, |
| "epoch": 0.09898962315674495, |
| "grad_norm": 0.06695123016834259, |
| "kl": 0.008675731522089336, |
| "learning_rate": 1.4473684210526317e-06, |
| "loss": 0.0001, |
| "num_tokens": 2159899.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.298653781414032, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 725 |
| }, |
| { |
| "clip_ratio": 0.000315199518809095, |
| "epoch": 0.09912616056799563, |
| "grad_norm": 0.0669025257229805, |
| "kl": 0.008661710817250423, |
| "learning_rate": 1.442105263157895e-06, |
| "loss": 0.0001, |
| "step": 726 |
| }, |
| { |
| "clip_ratio": 0.000989963358733803, |
| "epoch": 0.09926269797924632, |
| "grad_norm": 0.06626085191965103, |
| "kl": 0.008643930937978439, |
| "learning_rate": 1.4368421052631582e-06, |
| "loss": -0.0002, |
| "step": 727 |
| }, |
| { |
| "clip_ratio": 0.0006190385174704716, |
| "epoch": 0.099399235390497, |
| "grad_norm": 0.0670519545674324, |
| "kl": 0.008652786214952357, |
| "learning_rate": 1.4315789473684212e-06, |
| "loss": -0.0003, |
| "step": 728 |
| }, |
| { |
| "clip_ratio": 0.000202922077733092, |
| "completion_length": 123.84375, |
| "epoch": 0.09953577280174768, |
| "grad_norm": 0.033517949283123016, |
| "kl": 0.01238037904840894, |
| "learning_rate": 1.4263157894736843e-06, |
| "loss": -0.0, |
| "num_tokens": 2171070.0, |
| "reward": 0.887499988079071, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.887499988079071, |
| "step": 729 |
| }, |
| { |
| "clip_ratio": 0.00044238733244128525, |
| "epoch": 0.09967231021299836, |
| "grad_norm": 0.03357778489589691, |
| "kl": 0.012623131988220848, |
| "learning_rate": 1.4210526315789475e-06, |
| "loss": 0.0001, |
| "step": 730 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.09980884762424905, |
| "grad_norm": 0.03235095366835594, |
| "kl": 0.012562195843202062, |
| "learning_rate": 1.4157894736842107e-06, |
| "loss": -0.0, |
| "step": 731 |
| }, |
| { |
| "clip_ratio": 0.00035511364694684744, |
| "epoch": 0.09994538503549973, |
| "grad_norm": 0.03285340592265129, |
| "kl": 0.012679808860411867, |
| "learning_rate": 1.4105263157894738e-06, |
| "loss": 0.0, |
| "step": 732 |
| }, |
| { |
| "clip_ratio": 0.0004649055772460997, |
| "completion_length": 134.40625, |
| "epoch": 0.1000819224467504, |
| "grad_norm": 0.06454437971115112, |
| "kl": 0.011909478802408557, |
| "learning_rate": 1.405263157894737e-06, |
| "loss": 0.0001, |
| "num_tokens": 2182903.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 733 |
| }, |
| { |
| "clip_ratio": 0.0004135038034291938, |
| "epoch": 0.1002184598580011, |
| "grad_norm": 0.06428350508213043, |
| "kl": 0.011997650130069815, |
| "learning_rate": 1.4000000000000001e-06, |
| "loss": 0.0001, |
| "step": 734 |
| }, |
| { |
| "clip_ratio": 0.0002561475266702473, |
| "epoch": 0.10035499726925177, |
| "grad_norm": 0.06415695697069168, |
| "kl": 0.011796158534707502, |
| "learning_rate": 1.394736842105263e-06, |
| "loss": 0.0001, |
| "step": 735 |
| }, |
| { |
| "clip_ratio": 0.0003054668050026521, |
| "epoch": 0.10049153468050245, |
| "grad_norm": 0.06593475490808487, |
| "kl": 0.011560865576029755, |
| "learning_rate": 1.3894736842105263e-06, |
| "loss": -0.0001, |
| "step": 736 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 120.125, |
| "epoch": 0.10062807209175315, |
| "grad_norm": 0.02941080741584301, |
| "kl": 0.010089936651638709, |
| "learning_rate": 1.3842105263157896e-06, |
| "loss": -0.0, |
| "num_tokens": 2194655.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 737 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10076460950300382, |
| "grad_norm": 0.031720954924821854, |
| "kl": 0.009837967227213085, |
| "learning_rate": 1.3789473684210528e-06, |
| "loss": -0.0001, |
| "step": 738 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1009011469142545, |
| "grad_norm": 0.03245928883552551, |
| "kl": 0.010079732717713341, |
| "learning_rate": 1.373684210526316e-06, |
| "loss": -0.0002, |
| "step": 739 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1010376843255052, |
| "grad_norm": 0.03180951997637749, |
| "kl": 0.010071169104776345, |
| "learning_rate": 1.3684210526315791e-06, |
| "loss": -0.0001, |
| "step": 740 |
| }, |
| { |
| "clip_ratio": 0.001492358758696355, |
| "completion_length": 142.6875, |
| "epoch": 0.10117422173675587, |
| "grad_norm": 0.06009500473737717, |
| "kl": 0.009380554074596148, |
| "learning_rate": 1.3631578947368423e-06, |
| "loss": 0.0001, |
| "num_tokens": 2206217.0, |
| "reward": 0.5218750238418579, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.5218750238418579, |
| "step": 741 |
| }, |
| { |
| "clip_ratio": 0.0013410507890512235, |
| "epoch": 0.10131075914800655, |
| "grad_norm": 0.04778634384274483, |
| "kl": 0.009714217128930613, |
| "learning_rate": 1.3578947368421052e-06, |
| "loss": 0.0, |
| "step": 742 |
| }, |
| { |
| "clip_ratio": 0.0007410717880702578, |
| "epoch": 0.10144729655925723, |
| "grad_norm": 0.06011098250746727, |
| "kl": 0.009305261242843699, |
| "learning_rate": 1.3526315789473684e-06, |
| "loss": 0.0001, |
| "step": 743 |
| }, |
| { |
| "clip_ratio": 0.0014551018830388784, |
| "epoch": 0.10158383397050792, |
| "grad_norm": 0.059762001037597656, |
| "kl": 0.00951635251112748, |
| "learning_rate": 1.3473684210526316e-06, |
| "loss": 0.0001, |
| "step": 744 |
| }, |
| { |
| "clip_ratio": 0.0006466649356298149, |
| "completion_length": 144.21875, |
| "epoch": 0.1017203713817586, |
| "grad_norm": 0.0572526790201664, |
| "kl": 0.008839150817948394, |
| "learning_rate": 1.342105263157895e-06, |
| "loss": 0.0001, |
| "num_tokens": 2217876.0, |
| "reward": 0.296875, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.296875, |
| "step": 745 |
| }, |
| { |
| "clip_ratio": 0.0008269171958090737, |
| "epoch": 0.10185690879300928, |
| "grad_norm": 0.05735499784350395, |
| "kl": 0.008949853538069874, |
| "learning_rate": 1.3368421052631581e-06, |
| "loss": -0.0002, |
| "step": 746 |
| }, |
| { |
| "clip_ratio": 0.0004257605905877426, |
| "epoch": 0.10199344620425997, |
| "grad_norm": 0.05705785006284714, |
| "kl": 0.008868573815561831, |
| "learning_rate": 1.3315789473684213e-06, |
| "loss": 0.0001, |
| "step": 747 |
| }, |
| { |
| "clip_ratio": 0.00047347064537461847, |
| "epoch": 0.10212998361551065, |
| "grad_norm": 0.05702279135584831, |
| "kl": 0.008794885477982461, |
| "learning_rate": 1.3263157894736844e-06, |
| "loss": -0.0002, |
| "step": 748 |
| }, |
| { |
| "clip_ratio": 0.0005040322430431843, |
| "completion_length": 118.71875, |
| "epoch": 0.10226652102676133, |
| "grad_norm": 0.05455613136291504, |
| "kl": 0.01122648630553158, |
| "learning_rate": 1.3210526315789474e-06, |
| "loss": 0.0002, |
| "num_tokens": 2229051.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 749 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10240305843801202, |
| "grad_norm": 0.054574187844991684, |
| "kl": 0.01155937732255552, |
| "learning_rate": 1.3157894736842106e-06, |
| "loss": 0.0002, |
| "step": 750 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.1025395958492627, |
| "grad_norm": 0.05427771806716919, |
| "kl": 0.011250461713643745, |
| "learning_rate": 1.3105263157894737e-06, |
| "loss": 0.0001, |
| "step": 751 |
| }, |
| { |
| "clip_ratio": 0.00027173911803402007, |
| "epoch": 0.10267613326051338, |
| "grad_norm": 0.05351484566926956, |
| "kl": 0.011241496373258997, |
| "learning_rate": 1.3052631578947369e-06, |
| "loss": 0.0001, |
| "step": 752 |
| }, |
| { |
| "clip_ratio": 0.0005952381034148857, |
| "completion_length": 113.96875, |
| "epoch": 0.10281267067176407, |
| "grad_norm": 0.07776264101266861, |
| "kl": 0.010354362588259391, |
| "learning_rate": 1.3e-06, |
| "loss": 0.0003, |
| "num_tokens": 2240370.0, |
| "reward": 0.6906249523162842, |
| "reward_std": 0.36360570788383484, |
| "rewards/reward_fn": 0.6906249523162842, |
| "step": 753 |
| }, |
| { |
| "clip_ratio": 0.0003906250058207661, |
| "epoch": 0.10294920808301475, |
| "grad_norm": 0.07834309339523315, |
| "kl": 0.010167162035941146, |
| "learning_rate": 1.2947368421052634e-06, |
| "loss": 0.0002, |
| "step": 754 |
| }, |
| { |
| "clip_ratio": 0.0003906250058207661, |
| "epoch": 0.10308574549426543, |
| "grad_norm": 0.07701864093542099, |
| "kl": 0.010121552215423435, |
| "learning_rate": 1.2894736842105266e-06, |
| "loss": 0.0002, |
| "step": 755 |
| }, |
| { |
| "clip_ratio": 0.001165050794952549, |
| "epoch": 0.1032222829055161, |
| "grad_norm": 0.07689879834651947, |
| "kl": 0.009859664540272206, |
| "learning_rate": 1.2842105263157895e-06, |
| "loss": -0.0001, |
| "step": 756 |
| }, |
| { |
| "clip_ratio": 0.0006702122191200033, |
| "completion_length": 127.75, |
| "epoch": 0.1033588203167668, |
| "grad_norm": 0.04305388778448105, |
| "kl": 0.010872000711970031, |
| "learning_rate": 1.2789473684210527e-06, |
| "loss": -0.0001, |
| "num_tokens": 2252014.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 757 |
| }, |
| { |
| "clip_ratio": 0.000793718674685806, |
| "epoch": 0.10349535772801748, |
| "grad_norm": 0.046323131769895554, |
| "kl": 0.010926326060143765, |
| "learning_rate": 1.2736842105263159e-06, |
| "loss": -0.0, |
| "step": 758 |
| }, |
| { |
| "clip_ratio": 0.00042796415800694376, |
| "epoch": 0.10363189513926815, |
| "grad_norm": 0.043383169919252396, |
| "kl": 0.011011771603079978, |
| "learning_rate": 1.268421052631579e-06, |
| "loss": -0.0, |
| "step": 759 |
| }, |
| { |
| "clip_ratio": 0.00042796415800694376, |
| "epoch": 0.10376843255051885, |
| "grad_norm": 0.0435534343123436, |
| "kl": 0.011148589983349666, |
| "learning_rate": 1.2631578947368422e-06, |
| "loss": -0.0001, |
| "step": 760 |
| }, |
| { |
| "clip_ratio": 0.00045289855916053057, |
| "completion_length": 115.375, |
| "epoch": 0.10390496996176952, |
| "grad_norm": 0.06452547013759613, |
| "kl": 0.011318336823023856, |
| "learning_rate": 1.2578947368421054e-06, |
| "loss": 0.0001, |
| "num_tokens": 2262914.0, |
| "reward": 0.4937500059604645, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.4937500059604645, |
| "step": 761 |
| }, |
| { |
| "clip_ratio": 0.00045289855916053057, |
| "epoch": 0.1040415073730202, |
| "grad_norm": 0.06416850537061691, |
| "kl": 0.011546772257133853, |
| "learning_rate": 1.2526315789473687e-06, |
| "loss": 0.0, |
| "step": 762 |
| }, |
| { |
| "clip_ratio": 0.0008846690761856735, |
| "epoch": 0.1041780447842709, |
| "grad_norm": 0.06353920698165894, |
| "kl": 0.01127580660249805, |
| "learning_rate": 1.2473684210526317e-06, |
| "loss": 0.0002, |
| "step": 763 |
| }, |
| { |
| "clip_ratio": 0.0009263834217563272, |
| "epoch": 0.10431458219552157, |
| "grad_norm": 0.06321839988231659, |
| "kl": 0.011213135883735958, |
| "learning_rate": 1.2421052631578948e-06, |
| "loss": 0.0, |
| "step": 764 |
| }, |
| { |
| "clip_ratio": 0.00028935185400769114, |
| "completion_length": 114.21875, |
| "epoch": 0.10445111960677225, |
| "grad_norm": 0.039898358285427094, |
| "kl": 0.01145548326894641, |
| "learning_rate": 1.236842105263158e-06, |
| "loss": -0.0, |
| "num_tokens": 2273137.0, |
| "reward": 0.971875011920929, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.971875011920929, |
| "step": 765 |
| }, |
| { |
| "clip_ratio": 0.00028935185400769114, |
| "epoch": 0.10458765701802294, |
| "grad_norm": 0.039807628840208054, |
| "kl": 0.011456414693384431, |
| "learning_rate": 1.2315789473684212e-06, |
| "loss": 0.0001, |
| "step": 766 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10472419442927362, |
| "grad_norm": 0.03721645846962929, |
| "kl": 0.01143070675607305, |
| "learning_rate": 1.2263157894736843e-06, |
| "loss": 0.0001, |
| "step": 767 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1048607318405243, |
| "grad_norm": 0.03797837346792221, |
| "kl": 0.011435232430812903, |
| "learning_rate": 1.2210526315789475e-06, |
| "loss": 0.0, |
| "step": 768 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 137.78125, |
| "epoch": 0.10499726925177498, |
| "grad_norm": 0.02658233977854252, |
| "kl": 0.008618613399448805, |
| "learning_rate": 1.2157894736842107e-06, |
| "loss": 0.0001, |
| "num_tokens": 2284654.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 769 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10513380666302567, |
| "grad_norm": 0.027051806449890137, |
| "kl": 0.008507974271196872, |
| "learning_rate": 1.2105263157894738e-06, |
| "loss": 0.0001, |
| "step": 770 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10527034407427635, |
| "grad_norm": 0.0269402414560318, |
| "kl": 0.00845086978370091, |
| "learning_rate": 1.205263157894737e-06, |
| "loss": 0.0, |
| "step": 771 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10540688148552703, |
| "grad_norm": 0.026533640921115875, |
| "kl": 0.008608131465734914, |
| "learning_rate": 1.2000000000000002e-06, |
| "loss": 0.0, |
| "step": 772 |
| }, |
| { |
| "clip_ratio": 0.0003124999930150807, |
| "completion_length": 101.8125, |
| "epoch": 0.10554341889677772, |
| "grad_norm": 0.043257467448711395, |
| "kl": 0.010255538712954149, |
| "learning_rate": 1.1947368421052633e-06, |
| "loss": -0.0, |
| "num_tokens": 2295264.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 773 |
| }, |
| { |
| "clip_ratio": 0.0003124999930150807, |
| "epoch": 0.1056799563080284, |
| "grad_norm": 0.04238433018326759, |
| "kl": 0.010612577418214642, |
| "learning_rate": 1.1894736842105265e-06, |
| "loss": 0.0001, |
| "step": 774 |
| }, |
| { |
| "clip_ratio": 0.0003124999930150807, |
| "epoch": 0.10581649371927908, |
| "grad_norm": 0.04066017270088196, |
| "kl": 0.010311985679436475, |
| "learning_rate": 1.1842105263157894e-06, |
| "loss": -0.0, |
| "step": 775 |
| }, |
| { |
| "clip_ratio": 0.000538949272595346, |
| "epoch": 0.10595303113052977, |
| "grad_norm": 0.040270913392305374, |
| "kl": 0.010246703568554949, |
| "learning_rate": 1.1789473684210526e-06, |
| "loss": -0.0001, |
| "step": 776 |
| }, |
| { |
| "clip_ratio": 0.0003925960336346179, |
| "completion_length": 112.65625, |
| "epoch": 0.10608956854178045, |
| "grad_norm": 0.03344862908124924, |
| "kl": 0.010528261889703572, |
| "learning_rate": 1.173684210526316e-06, |
| "loss": 0.0001, |
| "num_tokens": 2305925.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 777 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10622610595303113, |
| "grad_norm": 0.03316456452012062, |
| "kl": 0.010777023184346035, |
| "learning_rate": 1.1684210526315791e-06, |
| "loss": 0.0, |
| "step": 778 |
| }, |
| { |
| "clip_ratio": 0.0004919141792925075, |
| "epoch": 0.10636264336428182, |
| "grad_norm": 0.03301002457737923, |
| "kl": 0.010544149205088615, |
| "learning_rate": 1.163157894736842e-06, |
| "loss": -0.0001, |
| "step": 779 |
| }, |
| { |
| "clip_ratio": 0.00012550200335681438, |
| "epoch": 0.1064991807755325, |
| "grad_norm": 0.03319751098752022, |
| "kl": 0.010344347363570705, |
| "learning_rate": 1.1578947368421053e-06, |
| "loss": 0.0001, |
| "step": 780 |
| }, |
| { |
| "clip_ratio": 0.0003430538054089993, |
| "completion_length": 147.0625, |
| "epoch": 0.10663571818678318, |
| "grad_norm": 0.05283012241125107, |
| "kl": 0.008913551682780962, |
| "learning_rate": 1.1526315789473686e-06, |
| "loss": -0.0, |
| "num_tokens": 2318187.0, |
| "reward": 0.578125, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.578125, |
| "step": 781 |
| }, |
| { |
| "clip_ratio": 0.0003396739193703979, |
| "epoch": 0.10677225559803386, |
| "grad_norm": 0.054724764078855515, |
| "kl": 0.008723079881747253, |
| "learning_rate": 1.1473684210526316e-06, |
| "loss": 0.0, |
| "step": 782 |
| }, |
| { |
| "clip_ratio": 0.0006241834198590368, |
| "epoch": 0.10690879300928455, |
| "grad_norm": 0.05417024344205856, |
| "kl": 0.008669076938531362, |
| "learning_rate": 1.1421052631578947e-06, |
| "loss": 0.0001, |
| "step": 783 |
| }, |
| { |
| "clip_ratio": 0.00013646288425661623, |
| "epoch": 0.10704533042053523, |
| "grad_norm": 0.053411029279232025, |
| "kl": 0.00885024315357441, |
| "learning_rate": 1.136842105263158e-06, |
| "loss": -0.0001, |
| "step": 784 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 125.25, |
| "epoch": 0.1071818678317859, |
| "grad_norm": 0.044166672974824905, |
| "kl": 0.010470507600985002, |
| "learning_rate": 1.1315789473684213e-06, |
| "loss": 0.0001, |
| "num_tokens": 2329491.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 785 |
| }, |
| { |
| "clip_ratio": 0.00041019324271474034, |
| "epoch": 0.1073184052430366, |
| "grad_norm": 0.04360298812389374, |
| "kl": 0.010632296318362933, |
| "learning_rate": 1.1263157894736842e-06, |
| "loss": 0.0001, |
| "step": 786 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10745494265428727, |
| "grad_norm": 0.0443100780248642, |
| "kl": 0.010563089097558986, |
| "learning_rate": 1.1210526315789474e-06, |
| "loss": 0.0, |
| "step": 787 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10759148006553795, |
| "grad_norm": 0.04373029246926308, |
| "kl": 0.01074617158883484, |
| "learning_rate": 1.1157894736842106e-06, |
| "loss": -0.0001, |
| "step": 788 |
| }, |
| { |
| "completion_length": 119.6875, |
| "epoch": 0.10772801747678865, |
| "grad_norm": 0.0, |
| "learning_rate": 1.110526315789474e-06, |
| "loss": 0.0, |
| "num_tokens": 2339681.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.0, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 789 |
| }, |
| { |
| "epoch": 0.10786455488803932, |
| "grad_norm": 0.0, |
| "learning_rate": 1.1052631578947369e-06, |
| "loss": 0.0, |
| "step": 790 |
| }, |
| { |
| "epoch": 0.10800109229929, |
| "grad_norm": 0.0, |
| "learning_rate": 1.1e-06, |
| "loss": 0.0, |
| "step": 791 |
| }, |
| { |
| "epoch": 0.1081376297105407, |
| "grad_norm": 0.0, |
| "learning_rate": 1.0947368421052632e-06, |
| "loss": 0.0, |
| "step": 792 |
| }, |
| { |
| "clip_ratio": 0.0005744949594372883, |
| "completion_length": 123.78125, |
| "epoch": 0.10827416712179137, |
| "grad_norm": 0.06851465255022049, |
| "kl": 0.009653476590756327, |
| "learning_rate": 1.0894736842105264e-06, |
| "loss": -0.0001, |
| "num_tokens": 2351014.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 793 |
| }, |
| { |
| "clip_ratio": 0.001084136194549501, |
| "epoch": 0.10841070453304205, |
| "grad_norm": 0.0720347911119461, |
| "kl": 0.00959034822153626, |
| "learning_rate": 1.0842105263157895e-06, |
| "loss": 0.0002, |
| "step": 794 |
| }, |
| { |
| "clip_ratio": 0.00015470296784769744, |
| "epoch": 0.10854724194429273, |
| "grad_norm": 0.06908923387527466, |
| "kl": 0.009725141506351065, |
| "learning_rate": 1.0789473684210527e-06, |
| "loss": -0.0, |
| "step": 795 |
| }, |
| { |
| "clip_ratio": 0.00015470296784769744, |
| "epoch": 0.10868377935554342, |
| "grad_norm": 0.07157684117555618, |
| "kl": 0.009408918704139069, |
| "learning_rate": 1.0736842105263159e-06, |
| "loss": 0.0003, |
| "step": 796 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.21875, |
| "epoch": 0.1088203167667941, |
| "grad_norm": 0.038190700113773346, |
| "kl": 0.009965231030946597, |
| "learning_rate": 1.068421052631579e-06, |
| "loss": 0.0002, |
| "num_tokens": 2362869.0, |
| "reward": 0.578125, |
| "reward_std": 0.12120190262794495, |
| "rewards/reward_fn": 0.578125, |
| "step": 797 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10895685417804478, |
| "grad_norm": 0.03805210441350937, |
| "kl": 0.010003825402236544, |
| "learning_rate": 1.0631578947368422e-06, |
| "loss": -0.0, |
| "step": 798 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10909339158929547, |
| "grad_norm": 0.037355970591306686, |
| "kl": 0.010021371381299105, |
| "learning_rate": 1.0578947368421054e-06, |
| "loss": 0.0, |
| "step": 799 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.10922992900054615, |
| "grad_norm": 0.037908099591732025, |
| "kl": 0.00993045204813825, |
| "learning_rate": 1.0526315789473685e-06, |
| "loss": -0.0, |
| "step": 800 |
| }, |
| { |
| "clip_ratio": 0.00037362967850640416, |
| "completion_length": 132.96875, |
| "epoch": 0.10936646641179683, |
| "grad_norm": 0.03724236413836479, |
| "kl": 0.00914044175442541, |
| "learning_rate": 1.0473684210526317e-06, |
| "loss": 0.0, |
| "num_tokens": 2374032.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 801 |
| }, |
| { |
| "clip_ratio": 0.0005077498499304056, |
| "epoch": 0.10950300382304752, |
| "grad_norm": 0.0377136766910553, |
| "kl": 0.009023394632095005, |
| "learning_rate": 1.0421052631578949e-06, |
| "loss": 0.0001, |
| "step": 802 |
| }, |
| { |
| "clip_ratio": 0.0004972146562067792, |
| "epoch": 0.1096395412342982, |
| "grad_norm": 0.03748948499560356, |
| "kl": 0.009087913145776838, |
| "learning_rate": 1.036842105263158e-06, |
| "loss": 0.0001, |
| "step": 803 |
| }, |
| { |
| "clip_ratio": 0.0003630944847827777, |
| "epoch": 0.10977607864554888, |
| "grad_norm": 0.03790314868092537, |
| "kl": 0.009051808374351822, |
| "learning_rate": 1.0315789473684212e-06, |
| "loss": -0.0, |
| "step": 804 |
| }, |
| { |
| "clip_ratio": 0.00017361111531499773, |
| "completion_length": 134.53125, |
| "epoch": 0.10991261605679957, |
| "grad_norm": 0.020981159061193466, |
| "kl": 0.008830284561554436, |
| "learning_rate": 1.0263157894736843e-06, |
| "loss": 0.0001, |
| "num_tokens": 2385597.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 805 |
| }, |
| { |
| "clip_ratio": 0.0002367424312978983, |
| "epoch": 0.11004915346805025, |
| "grad_norm": 0.02120996080338955, |
| "kl": 0.009051185734278988, |
| "learning_rate": 1.0210526315789475e-06, |
| "loss": 0.0001, |
| "step": 806 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11018569087930093, |
| "grad_norm": 0.020987244322896004, |
| "kl": 0.008791206433670595, |
| "learning_rate": 1.0157894736842105e-06, |
| "loss": 0.0001, |
| "step": 807 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1103222282905516, |
| "grad_norm": 0.02127201482653618, |
| "kl": 0.008922349828935694, |
| "learning_rate": 1.0105263157894738e-06, |
| "loss": 0.0001, |
| "step": 808 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 138.5625, |
| "epoch": 0.1104587657018023, |
| "grad_norm": 0.059530384838581085, |
| "kl": 0.01224294574058149, |
| "learning_rate": 1.005263157894737e-06, |
| "loss": 0.0001, |
| "num_tokens": 2396915.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 809 |
| }, |
| { |
| "clip_ratio": 0.0004946099215885624, |
| "epoch": 0.11059530311305298, |
| "grad_norm": 0.059865981340408325, |
| "kl": 0.01249943784205243, |
| "learning_rate": 1.0000000000000002e-06, |
| "loss": 0.0003, |
| "step": 810 |
| }, |
| { |
| "clip_ratio": 0.00026483050896786153, |
| "epoch": 0.11073184052430365, |
| "grad_norm": 0.05934388190507889, |
| "kl": 0.01243184653139906, |
| "learning_rate": 9.947368421052631e-07, |
| "loss": -0.0, |
| "step": 811 |
| }, |
| { |
| "clip_ratio": 0.0002367424312978983, |
| "epoch": 0.11086837793555435, |
| "grad_norm": 0.05951414629817009, |
| "kl": 0.012573402127600275, |
| "learning_rate": 9.894736842105265e-07, |
| "loss": -0.0001, |
| "step": 812 |
| }, |
| { |
| "clip_ratio": 0.00022482014901470393, |
| "completion_length": 119.1875, |
| "epoch": 0.11100491534680502, |
| "grad_norm": 0.08948002755641937, |
| "kl": 0.010440508398460224, |
| "learning_rate": 9.842105263157897e-07, |
| "loss": 0.0002, |
| "num_tokens": 2408065.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.35490381717681885, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 813 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1111414527580557, |
| "grad_norm": 0.08962646126747131, |
| "kl": 0.010390780917077791, |
| "learning_rate": 9.789473684210526e-07, |
| "loss": 0.0002, |
| "step": 814 |
| }, |
| { |
| "clip_ratio": 0.00020161290012765676, |
| "epoch": 0.1112779901693064, |
| "grad_norm": 0.08933047950267792, |
| "kl": 0.010122088053321932, |
| "learning_rate": 9.736842105263158e-07, |
| "loss": -0.0, |
| "step": 815 |
| }, |
| { |
| "clip_ratio": 0.00044642857392318547, |
| "epoch": 0.11141452758055707, |
| "grad_norm": 0.0889420211315155, |
| "kl": 0.010298235552909318, |
| "learning_rate": 9.68421052631579e-07, |
| "loss": -0.0004, |
| "step": 816 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.21875, |
| "epoch": 0.11155106499180775, |
| "grad_norm": 0.04320777580142021, |
| "kl": 0.009522488442598842, |
| "learning_rate": 9.631578947368423e-07, |
| "loss": 0.0001, |
| "num_tokens": 2418728.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 817 |
| }, |
| { |
| "clip_ratio": 0.00014671361714135855, |
| "epoch": 0.11168760240305844, |
| "grad_norm": 0.04320985823869705, |
| "kl": 0.009407084122358356, |
| "learning_rate": 9.578947368421053e-07, |
| "loss": 0.0001, |
| "step": 818 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11182413981430912, |
| "grad_norm": 0.042713869363069534, |
| "kl": 0.009065183017810341, |
| "learning_rate": 9.526315789473685e-07, |
| "loss": -0.0001, |
| "step": 819 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1119606772255598, |
| "grad_norm": 0.04328214004635811, |
| "kl": 0.00947425660706358, |
| "learning_rate": 9.473684210526317e-07, |
| "loss": 0.0001, |
| "step": 820 |
| }, |
| { |
| "clip_ratio": 0.0009997597226174548, |
| "completion_length": 129.96875, |
| "epoch": 0.11209721463681048, |
| "grad_norm": 0.04568544030189514, |
| "kl": 0.00900749161519343, |
| "learning_rate": 9.421052631578948e-07, |
| "loss": 0.0, |
| "num_tokens": 2429911.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 821 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11223375204806117, |
| "grad_norm": 0.04508956894278526, |
| "kl": 0.009192549223371316, |
| "learning_rate": 9.368421052631579e-07, |
| "loss": -0.0001, |
| "step": 822 |
| }, |
| { |
| "clip_ratio": 0.0004810696409549564, |
| "epoch": 0.11237028945931185, |
| "grad_norm": 0.04525609314441681, |
| "kl": 0.009055977468960918, |
| "learning_rate": 9.315789473684212e-07, |
| "loss": -0.0002, |
| "step": 823 |
| }, |
| { |
| "clip_ratio": 0.00030014856019988656, |
| "epoch": 0.11250682687056253, |
| "grad_norm": 0.04554551839828491, |
| "kl": 0.009097619375097565, |
| "learning_rate": 9.263157894736844e-07, |
| "loss": -0.0002, |
| "step": 824 |
| }, |
| { |
| "clip_ratio": 0.00035919540096074343, |
| "completion_length": 104.59375, |
| "epoch": 0.11264336428181322, |
| "grad_norm": 0.02620200254023075, |
| "kl": 0.009858106132014655, |
| "learning_rate": 9.210526315789474e-07, |
| "loss": 0.0, |
| "num_tokens": 2440738.0, |
| "reward": 0.9156249761581421, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.9156249761581421, |
| "step": 825 |
| }, |
| { |
| "clip_ratio": 0.00035919540096074343, |
| "epoch": 0.1127799016930639, |
| "grad_norm": 0.025711508467793465, |
| "kl": 0.009933086832461413, |
| "learning_rate": 9.157894736842106e-07, |
| "loss": 0.0, |
| "step": 826 |
| }, |
| { |
| "clip_ratio": 0.0006780729454476386, |
| "epoch": 0.11291643910431458, |
| "grad_norm": 0.026838209480047226, |
| "kl": 0.009670235856901854, |
| "learning_rate": 9.105263157894737e-07, |
| "loss": 0.0001, |
| "step": 827 |
| }, |
| { |
| "clip_ratio": 0.00035919540096074343, |
| "epoch": 0.11305297651556527, |
| "grad_norm": 0.025257034227252007, |
| "kl": 0.010027570635429583, |
| "learning_rate": 9.05263157894737e-07, |
| "loss": 0.0, |
| "step": 828 |
| }, |
| { |
| "clip_ratio": 0.000797146960394457, |
| "completion_length": 122.25, |
| "epoch": 0.11318951392681595, |
| "grad_norm": 0.07018585503101349, |
| "kl": 0.009983684445614927, |
| "learning_rate": 9.000000000000001e-07, |
| "loss": 0.0, |
| "num_tokens": 2451310.0, |
| "reward": 0.6343750357627869, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.6343750357627869, |
| "step": 829 |
| }, |
| { |
| "clip_ratio": 0.0006165111262816936, |
| "epoch": 0.11332605133806663, |
| "grad_norm": 0.06987402588129044, |
| "kl": 0.009894670649373438, |
| "learning_rate": 8.947368421052632e-07, |
| "loss": 0.0001, |
| "step": 830 |
| }, |
| { |
| "clip_ratio": 0.0006148156535346061, |
| "epoch": 0.11346258874931732, |
| "grad_norm": 0.06946758180856705, |
| "kl": 0.009832889743847772, |
| "learning_rate": 8.894736842105264e-07, |
| "loss": -0.0001, |
| "step": 831 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.113599126160568, |
| "grad_norm": 0.07016166299581528, |
| "kl": 0.009600598204997368, |
| "learning_rate": 8.842105263157895e-07, |
| "loss": -0.0003, |
| "step": 832 |
| }, |
| { |
| "clip_ratio": 0.0011313995491946116, |
| "completion_length": 146.59375, |
| "epoch": 0.11373566357181868, |
| "grad_norm": 0.07958237826824188, |
| "kl": 0.009611170455173124, |
| "learning_rate": 8.789473684210527e-07, |
| "loss": 0.0003, |
| "num_tokens": 2463401.0, |
| "reward": 0.6343750357627869, |
| "reward_std": 0.4285576343536377, |
| "rewards/reward_fn": 0.6343750357627869, |
| "step": 833 |
| }, |
| { |
| "clip_ratio": 0.0016153024625964463, |
| "epoch": 0.11387220098306935, |
| "grad_norm": 0.07779110968112946, |
| "kl": 0.009133476429269649, |
| "learning_rate": 8.736842105263159e-07, |
| "loss": 0.0003, |
| "step": 834 |
| }, |
| { |
| "clip_ratio": 0.0010061696957563981, |
| "epoch": 0.11400873839432005, |
| "grad_norm": 0.0776069313287735, |
| "kl": 0.00904379173880443, |
| "learning_rate": 8.68421052631579e-07, |
| "loss": 0.0003, |
| "step": 835 |
| }, |
| { |
| "clip_ratio": 0.0007421910850098357, |
| "epoch": 0.11414527580557073, |
| "grad_norm": 0.07728521525859833, |
| "kl": 0.009365054698719177, |
| "learning_rate": 8.631578947368421e-07, |
| "loss": 0.0002, |
| "step": 836 |
| }, |
| { |
| "clip_ratio": 0.0010523088785703294, |
| "completion_length": 123.59375, |
| "epoch": 0.1142818132168214, |
| "grad_norm": 0.03648988530039787, |
| "kl": 0.008774209381954279, |
| "learning_rate": 8.578947368421054e-07, |
| "loss": 0.0003, |
| "num_tokens": 2474472.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 837 |
| }, |
| { |
| "clip_ratio": 0.0007918921983218752, |
| "epoch": 0.1144183506280721, |
| "grad_norm": 0.036389607936143875, |
| "kl": 0.008319388878589962, |
| "learning_rate": 8.526315789473685e-07, |
| "loss": 0.0003, |
| "step": 838 |
| }, |
| { |
| "clip_ratio": 0.00026709403027780354, |
| "epoch": 0.11455488803932277, |
| "grad_norm": 0.036154378205537796, |
| "kl": 0.008852859253238421, |
| "learning_rate": 8.473684210526316e-07, |
| "loss": 0.0002, |
| "step": 839 |
| }, |
| { |
| "clip_ratio": 0.0011004273837897927, |
| "epoch": 0.11469142545057345, |
| "grad_norm": 0.036567799746990204, |
| "kl": 0.008751676294195931, |
| "learning_rate": 8.421052631578948e-07, |
| "loss": 0.0002, |
| "step": 840 |
| }, |
| { |
| "clip_ratio": 0.0003676470660138875, |
| "completion_length": 120.21875, |
| "epoch": 0.11482796286182415, |
| "grad_norm": 0.05161119997501373, |
| "kl": 0.010281605340423994, |
| "learning_rate": 8.36842105263158e-07, |
| "loss": -0.0001, |
| "num_tokens": 2485419.0, |
| "reward": 0.6906249523162842, |
| "reward_std": 0.16874998807907104, |
| "rewards/reward_fn": 0.6906249523162842, |
| "step": 841 |
| }, |
| { |
| "clip_ratio": 0.0007912741857580841, |
| "epoch": 0.11496450027307482, |
| "grad_norm": 0.051767535507678986, |
| "kl": 0.01021325092006009, |
| "learning_rate": 8.315789473684212e-07, |
| "loss": 0.0002, |
| "step": 842 |
| }, |
| { |
| "clip_ratio": 0.00047323029139079154, |
| "epoch": 0.1151010376843255, |
| "grad_norm": 0.05189364403486252, |
| "kl": 0.009887757805699948, |
| "learning_rate": 8.263157894736843e-07, |
| "loss": -0.0, |
| "step": 843 |
| }, |
| { |
| "clip_ratio": 0.0008636788261355832, |
| "epoch": 0.1152375750955762, |
| "grad_norm": 0.05228329822421074, |
| "kl": 0.010114929849805776, |
| "learning_rate": 8.210526315789474e-07, |
| "loss": -0.0001, |
| "step": 844 |
| }, |
| { |
| "clip_ratio": 0.00024900399148464203, |
| "completion_length": 131.875, |
| "epoch": 0.11537411250682687, |
| "grad_norm": 0.055897049605846405, |
| "kl": 0.00992464779119473, |
| "learning_rate": 8.157894736842106e-07, |
| "loss": 0.0, |
| "num_tokens": 2497503.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 845 |
| }, |
| { |
| "clip_ratio": 0.0005582029843935743, |
| "epoch": 0.11551064991807755, |
| "grad_norm": 0.059186290949583054, |
| "kl": 0.009978658286854625, |
| "learning_rate": 8.105263157894736e-07, |
| "loss": -0.0001, |
| "step": 846 |
| }, |
| { |
| "clip_ratio": 0.0006280765373958275, |
| "epoch": 0.11564718732932823, |
| "grad_norm": 0.05966058745980263, |
| "kl": 0.009848977206274867, |
| "learning_rate": 8.052631578947369e-07, |
| "loss": 0.0, |
| "step": 847 |
| }, |
| { |
| "clip_ratio": 0.0005072684434708208, |
| "epoch": 0.11578372474057892, |
| "grad_norm": 0.057398419827222824, |
| "kl": 0.009751228448294569, |
| "learning_rate": 8.000000000000001e-07, |
| "loss": 0.0001, |
| "step": 848 |
| }, |
| { |
| "clip_ratio": 0.0007949806749820709, |
| "completion_length": 113.59375, |
| "epoch": 0.1159202621518296, |
| "grad_norm": 0.06578940153121948, |
| "kl": 0.010100882951519452, |
| "learning_rate": 7.947368421052632e-07, |
| "loss": 0.0, |
| "num_tokens": 2507770.0, |
| "reward": 0.578125, |
| "reward_std": 0.28125, |
| "rewards/reward_fn": 0.578125, |
| "step": 849 |
| }, |
| { |
| "clip_ratio": 0.0005584363534580916, |
| "epoch": 0.11605679956308028, |
| "grad_norm": 0.06668176501989365, |
| "kl": 0.010129980189958587, |
| "learning_rate": 7.894736842105263e-07, |
| "loss": -0.0001, |
| "step": 850 |
| }, |
| { |
| "clip_ratio": 0.0005584363534580916, |
| "epoch": 0.11619333697433097, |
| "grad_norm": 0.06607290357351303, |
| "kl": 0.010091839678352699, |
| "learning_rate": 7.842105263157896e-07, |
| "loss": -0.0002, |
| "step": 851 |
| }, |
| { |
| "clip_ratio": 0.0005751371791120619, |
| "epoch": 0.11632987438558165, |
| "grad_norm": 0.06626007705926895, |
| "kl": 0.010161791986320168, |
| "learning_rate": 7.789473684210527e-07, |
| "loss": -0.0, |
| "step": 852 |
| }, |
| { |
| "clip_ratio": 0.00035511364694684744, |
| "completion_length": 110.09375, |
| "epoch": 0.11646641179683233, |
| "grad_norm": 0.038698773831129074, |
| "kl": 0.009669568818935659, |
| "learning_rate": 7.736842105263158e-07, |
| "loss": 0.0001, |
| "num_tokens": 2518237.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 853 |
| }, |
| { |
| "clip_ratio": 0.000540928915143013, |
| "epoch": 0.11660294920808302, |
| "grad_norm": 0.03822898492217064, |
| "kl": 0.009661782307375688, |
| "learning_rate": 7.684210526315789e-07, |
| "loss": 0.0001, |
| "step": 854 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1167394866193337, |
| "grad_norm": 0.03842921927571297, |
| "kl": 0.009424046482308768, |
| "learning_rate": 7.631578947368422e-07, |
| "loss": 0.0, |
| "step": 855 |
| }, |
| { |
| "clip_ratio": 0.000540928915143013, |
| "epoch": 0.11687602403058438, |
| "grad_norm": 0.03891470655798912, |
| "kl": 0.009642332544899546, |
| "learning_rate": 7.578947368421054e-07, |
| "loss": 0.0001, |
| "step": 856 |
| }, |
| { |
| "clip_ratio": 0.0001197317978949286, |
| "completion_length": 139.0, |
| "epoch": 0.11701256144183507, |
| "grad_norm": 0.04915855824947357, |
| "kl": 0.008732691851037089, |
| "learning_rate": 7.526315789473684e-07, |
| "loss": -0.0, |
| "num_tokens": 2530009.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 857 |
| }, |
| { |
| "clip_ratio": 0.000202922077733092, |
| "epoch": 0.11714909885308575, |
| "grad_norm": 0.04969601333141327, |
| "kl": 0.008386543231608812, |
| "learning_rate": 7.473684210526316e-07, |
| "loss": -0.0, |
| "step": 858 |
| }, |
| { |
| "clip_ratio": 0.00028669723542407155, |
| "epoch": 0.11728563626433643, |
| "grad_norm": 0.05026835575699806, |
| "kl": 0.008854215098835994, |
| "learning_rate": 7.421052631578949e-07, |
| "loss": 0.0001, |
| "step": 859 |
| }, |
| { |
| "clip_ratio": 0.0006337854138109833, |
| "epoch": 0.1174221736755871, |
| "grad_norm": 0.049797896295785904, |
| "kl": 0.00838919422676554, |
| "learning_rate": 7.368421052631579e-07, |
| "loss": -0.0, |
| "step": 860 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.875, |
| "epoch": 0.1175587110868378, |
| "grad_norm": 0.05981166660785675, |
| "kl": 0.010764563499833457, |
| "learning_rate": 7.315789473684211e-07, |
| "loss": 0.0002, |
| "num_tokens": 2540865.0, |
| "reward": 0.6625000238418579, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.6625000238418579, |
| "step": 861 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11769524849808848, |
| "grad_norm": 0.05998644605278969, |
| "kl": 0.010729881760198623, |
| "learning_rate": 7.263157894736843e-07, |
| "loss": 0.0002, |
| "step": 862 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11783178590933915, |
| "grad_norm": 0.06080706790089607, |
| "kl": 0.010663294306141324, |
| "learning_rate": 7.210526315789475e-07, |
| "loss": 0.0002, |
| "step": 863 |
| }, |
| { |
| "clip_ratio": 0.000480769231216982, |
| "epoch": 0.11796832332058985, |
| "grad_norm": 0.061566147953271866, |
| "kl": 0.01102626580541255, |
| "learning_rate": 7.157894736842106e-07, |
| "loss": 0.0002, |
| "step": 864 |
| }, |
| { |
| "clip_ratio": 0.0009241341904271394, |
| "completion_length": 132.40625, |
| "epoch": 0.11810486073184052, |
| "grad_norm": 0.057581037282943726, |
| "kl": 0.009243382985005155, |
| "learning_rate": 7.105263157894737e-07, |
| "loss": 0.0, |
| "num_tokens": 2552462.0, |
| "reward": 0.4937500059604645, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.4937500059604645, |
| "step": 865 |
| }, |
| { |
| "clip_ratio": 0.001048282952979207, |
| "epoch": 0.1182413981430912, |
| "grad_norm": 0.05778654292225838, |
| "kl": 0.009179890039376915, |
| "learning_rate": 7.052631578947369e-07, |
| "loss": 0.0002, |
| "step": 866 |
| }, |
| { |
| "clip_ratio": 0.0008596471889177337, |
| "epoch": 0.1183779355543419, |
| "grad_norm": 0.05647100880742073, |
| "kl": 0.009399849390320014, |
| "learning_rate": 7.000000000000001e-07, |
| "loss": 0.0002, |
| "step": 867 |
| }, |
| { |
| "clip_ratio": 0.0010773850663099438, |
| "epoch": 0.11851447296559257, |
| "grad_norm": 0.058462709188461304, |
| "kl": 0.00914568846928887, |
| "learning_rate": 6.947368421052631e-07, |
| "loss": -0.0, |
| "step": 868 |
| }, |
| { |
| "clip_ratio": 0.00034340660204179585, |
| "completion_length": 110.4375, |
| "epoch": 0.11865101037684325, |
| "grad_norm": 0.033963970839977264, |
| "kl": 0.008949848102929536, |
| "learning_rate": 6.894736842105264e-07, |
| "loss": 0.0002, |
| "num_tokens": 2562664.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 869 |
| }, |
| { |
| "clip_ratio": 0.0005132435617269948, |
| "epoch": 0.11878754778809394, |
| "grad_norm": 0.03405847027897835, |
| "kl": 0.008978909980214667, |
| "learning_rate": 6.842105263157896e-07, |
| "loss": 0.0002, |
| "step": 870 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11892408519934462, |
| "grad_norm": 0.03434007614850998, |
| "kl": 0.008946208741690498, |
| "learning_rate": 6.789473684210526e-07, |
| "loss": 0.0002, |
| "step": 871 |
| }, |
| { |
| "clip_ratio": 0.0002561475266702473, |
| "epoch": 0.1190606226105953, |
| "grad_norm": 0.0348217748105526, |
| "kl": 0.009055694528797176, |
| "learning_rate": 6.736842105263158e-07, |
| "loss": 0.0002, |
| "step": 872 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.6875, |
| "epoch": 0.11919716002184598, |
| "grad_norm": 0.03698798269033432, |
| "kl": 0.00979738227033522, |
| "learning_rate": 6.684210526315791e-07, |
| "loss": 0.0, |
| "num_tokens": 2574250.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 873 |
| }, |
| { |
| "clip_ratio": 0.000240384615608491, |
| "epoch": 0.11933369743309667, |
| "grad_norm": 0.03672180324792862, |
| "kl": 0.009651636610215064, |
| "learning_rate": 6.631578947368422e-07, |
| "loss": 0.0, |
| "step": 874 |
| }, |
| { |
| "clip_ratio": 0.00014204545004758984, |
| "epoch": 0.11947023484434735, |
| "grad_norm": 0.03639303147792816, |
| "kl": 0.009492714212683495, |
| "learning_rate": 6.578947368421053e-07, |
| "loss": 0.0001, |
| "step": 875 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.11960677225559803, |
| "grad_norm": 0.03633575886487961, |
| "kl": 0.009816416597459465, |
| "learning_rate": 6.526315789473684e-07, |
| "loss": 0.0, |
| "step": 876 |
| }, |
| { |
| "clip_ratio": 0.00033602150506339967, |
| "completion_length": 127.625, |
| "epoch": 0.11974330966684872, |
| "grad_norm": 0.07376956194639206, |
| "kl": 0.010965834895614535, |
| "learning_rate": 6.473684210526317e-07, |
| "loss": 0.0001, |
| "num_tokens": 2585166.0, |
| "reward": 0.8031249642372131, |
| "reward_std": 0.29865381121635437, |
| "rewards/reward_fn": 0.8031249642372131, |
| "step": 877 |
| }, |
| { |
| "clip_ratio": 0.0004446329839993268, |
| "epoch": 0.1198798470780994, |
| "grad_norm": 0.07392875850200653, |
| "kl": 0.01122175675118342, |
| "learning_rate": 6.421052631578948e-07, |
| "loss": -0.0001, |
| "step": 878 |
| }, |
| { |
| "clip_ratio": 0.001275275441003032, |
| "epoch": 0.12001638448935008, |
| "grad_norm": 0.07364289462566376, |
| "kl": 0.01099988129135454, |
| "learning_rate": 6.368421052631579e-07, |
| "loss": -0.0, |
| "step": 879 |
| }, |
| { |
| "clip_ratio": 0.0005145929317222908, |
| "epoch": 0.12015292190060077, |
| "grad_norm": 0.07408083230257034, |
| "kl": 0.011017758675734513, |
| "learning_rate": 6.315789473684211e-07, |
| "loss": 0.0, |
| "step": 880 |
| }, |
| { |
| "clip_ratio": 0.000577568804146722, |
| "completion_length": 133.75, |
| "epoch": 0.12028945931185145, |
| "grad_norm": 0.06023528054356575, |
| "kl": 0.01213556258880999, |
| "learning_rate": 6.263157894736844e-07, |
| "loss": 0.0001, |
| "num_tokens": 2596854.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.298653781414032, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 881 |
| }, |
| { |
| "clip_ratio": 0.0009233094315277413, |
| "epoch": 0.12042599672310213, |
| "grad_norm": 0.05947253853082657, |
| "kl": 0.012005855125607923, |
| "learning_rate": 6.210526315789474e-07, |
| "loss": 0.0001, |
| "step": 882 |
| }, |
| { |
| "clip_ratio": 0.0013617751974379644, |
| "epoch": 0.12056253413435282, |
| "grad_norm": 0.05955258756875992, |
| "kl": 0.012278351889108308, |
| "learning_rate": 6.157894736842106e-07, |
| "loss": 0.0001, |
| "step": 883 |
| }, |
| { |
| "clip_ratio": 0.0008541174756828696, |
| "epoch": 0.1206990715456035, |
| "grad_norm": 0.060061726719141006, |
| "kl": 0.01200440626416821, |
| "learning_rate": 6.105263157894738e-07, |
| "loss": 0.0002, |
| "step": 884 |
| }, |
| { |
| "clip_ratio": 0.00027901786961592734, |
| "completion_length": 107.6875, |
| "epoch": 0.12083560895685418, |
| "grad_norm": 0.050591591745615005, |
| "kl": 0.010622919769957662, |
| "learning_rate": 6.052631578947369e-07, |
| "loss": 0.0003, |
| "num_tokens": 2607468.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 885 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12097214636810485, |
| "grad_norm": 0.050506219267845154, |
| "kl": 0.010465682767971884, |
| "learning_rate": 6.000000000000001e-07, |
| "loss": 0.0001, |
| "step": 886 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12110868377935555, |
| "grad_norm": 0.050464119762182236, |
| "kl": 0.01065878513327334, |
| "learning_rate": 5.947368421052632e-07, |
| "loss": 0.0003, |
| "step": 887 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.12124522119060623, |
| "grad_norm": 0.050334133207798004, |
| "kl": 0.010576253538602032, |
| "learning_rate": 5.894736842105263e-07, |
| "loss": 0.0002, |
| "step": 888 |
| }, |
| { |
| "clip_ratio": 0.0002281021879753098, |
| "completion_length": 114.09375, |
| "epoch": 0.1213817586018569, |
| "grad_norm": 0.058507226407527924, |
| "kl": 0.009911755863868166, |
| "learning_rate": 5.842105263157896e-07, |
| "loss": -0.0, |
| "num_tokens": 2618455.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 889 |
| }, |
| { |
| "clip_ratio": 0.0006347758026095107, |
| "epoch": 0.1215182960131076, |
| "grad_norm": 0.05302344262599945, |
| "kl": 0.009213384124450386, |
| "learning_rate": 5.789473684210526e-07, |
| "loss": 0.0001, |
| "step": 890 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12165483342435827, |
| "grad_norm": 0.05901410058140755, |
| "kl": 0.00989249241683865, |
| "learning_rate": 5.736842105263158e-07, |
| "loss": 0.0, |
| "step": 891 |
| }, |
| { |
| "clip_ratio": 0.0002281021879753098, |
| "epoch": 0.12179137083560895, |
| "grad_norm": 0.05360320582985878, |
| "kl": 0.009214803438226227, |
| "learning_rate": 5.68421052631579e-07, |
| "loss": 0.0, |
| "step": 892 |
| }, |
| { |
| "clip_ratio": 0.0004417457530507818, |
| "completion_length": 121.03125, |
| "epoch": 0.12192790824685965, |
| "grad_norm": 0.06554637849330902, |
| "kl": 0.00874528284475673, |
| "learning_rate": 5.631578947368421e-07, |
| "loss": 0.0003, |
| "num_tokens": 2629012.0, |
| "reward": 0.40937498211860657, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.40937498211860657, |
| "step": 893 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12206444565811032, |
| "grad_norm": 0.06554420292377472, |
| "kl": 0.008677933998114895, |
| "learning_rate": 5.578947368421053e-07, |
| "loss": 0.0002, |
| "step": 894 |
| }, |
| { |
| "clip_ratio": 0.00021853146608918905, |
| "epoch": 0.122200983069361, |
| "grad_norm": 0.06595013290643692, |
| "kl": 0.008711180526006501, |
| "learning_rate": 5.526315789473684e-07, |
| "loss": 0.0002, |
| "step": 895 |
| }, |
| { |
| "clip_ratio": 0.00021853146608918905, |
| "epoch": 0.1223375204806117, |
| "grad_norm": 0.06584534049034119, |
| "kl": 0.008944936875195708, |
| "learning_rate": 5.473684210526316e-07, |
| "loss": 0.0002, |
| "step": 896 |
| }, |
| { |
| "clip_ratio": 0.00042743951780721545, |
| "completion_length": 126.96875, |
| "epoch": 0.12247405789186237, |
| "grad_norm": 0.03363471105694771, |
| "kl": 0.009414180800376926, |
| "learning_rate": 5.421052631578948e-07, |
| "loss": 0.0003, |
| "num_tokens": 2640279.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 897 |
| }, |
| { |
| "clip_ratio": 0.0007835259602870792, |
| "epoch": 0.12261059530311305, |
| "grad_norm": 0.03353031724691391, |
| "kl": 0.009427034252439626, |
| "learning_rate": 5.368421052631579e-07, |
| "loss": 0.0003, |
| "step": 898 |
| }, |
| { |
| "clip_ratio": 0.0005511907947948202, |
| "epoch": 0.12274713271436373, |
| "grad_norm": 0.03437519818544388, |
| "kl": 0.009330364089692011, |
| "learning_rate": 5.315789473684211e-07, |
| "loss": 0.0002, |
| "step": 899 |
| }, |
| { |
| "clip_ratio": 0.0007645831792615354, |
| "epoch": 0.12288367012561442, |
| "grad_norm": 0.03362502530217171, |
| "kl": 0.009262417835998349, |
| "learning_rate": 5.263157894736843e-07, |
| "loss": 0.0002, |
| "step": 900 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 122.9375, |
| "epoch": 0.1230202075368651, |
| "grad_norm": 0.04992664232850075, |
| "kl": 0.012523235520347953, |
| "learning_rate": 5.210526315789474e-07, |
| "loss": -0.0, |
| "num_tokens": 2651857.0, |
| "reward": 0.4937499761581421, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.4937499761581421, |
| "step": 901 |
| }, |
| { |
| "clip_ratio": 0.0003633720916695893, |
| "epoch": 0.12315674494811578, |
| "grad_norm": 0.0510023832321167, |
| "kl": 0.013408263184828684, |
| "learning_rate": 5.157894736842106e-07, |
| "loss": 0.0, |
| "step": 902 |
| }, |
| { |
| "clip_ratio": 0.0003731642500497401, |
| "epoch": 0.12329328235936647, |
| "grad_norm": 0.05082482844591141, |
| "kl": 0.013034153766056988, |
| "learning_rate": 5.105263157894738e-07, |
| "loss": 0.0002, |
| "step": 903 |
| }, |
| { |
| "clip_ratio": 0.00042346086411271244, |
| "epoch": 0.12342981977061715, |
| "grad_norm": 0.05140971764922142, |
| "kl": 0.012985301575099584, |
| "learning_rate": 5.052631578947369e-07, |
| "loss": -0.0, |
| "step": 904 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.5, |
| "epoch": 0.12356635718186783, |
| "grad_norm": 0.0390377938747406, |
| "kl": 0.01020817035168875, |
| "learning_rate": 5.000000000000001e-07, |
| "loss": 0.0002, |
| "num_tokens": 2662189.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 905 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12370289459311852, |
| "grad_norm": 0.03885956108570099, |
| "kl": 0.010147387016331777, |
| "learning_rate": 4.947368421052632e-07, |
| "loss": 0.0002, |
| "step": 906 |
| }, |
| { |
| "clip_ratio": 0.00020032051543239504, |
| "epoch": 0.1238394320043692, |
| "grad_norm": 0.03814854845404625, |
| "kl": 0.010349849821068347, |
| "learning_rate": 4.894736842105263e-07, |
| "loss": 0.0, |
| "step": 907 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12397596941561988, |
| "grad_norm": 0.03928220644593239, |
| "kl": 0.010089321731356904, |
| "learning_rate": 4.842105263157895e-07, |
| "loss": 0.0002, |
| "step": 908 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.375, |
| "epoch": 0.12411250682687057, |
| "grad_norm": 0.03833654522895813, |
| "kl": 0.010375823403592221, |
| "learning_rate": 4.789473684210526e-07, |
| "loss": 0.0001, |
| "num_tokens": 2672653.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 909 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12424904423812125, |
| "grad_norm": 0.03808671608567238, |
| "kl": 0.010178016913414467, |
| "learning_rate": 4.7368421052631585e-07, |
| "loss": 0.0001, |
| "step": 910 |
| }, |
| { |
| "clip_ratio": 0.0003955696302000433, |
| "epoch": 0.12438558164937193, |
| "grad_norm": 0.038504257798194885, |
| "kl": 0.010193661226367112, |
| "learning_rate": 4.6842105263157896e-07, |
| "loss": 0.0002, |
| "step": 911 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1245221190606226, |
| "grad_norm": 0.03888189047574997, |
| "kl": 0.010565640797722153, |
| "learning_rate": 4.631578947368422e-07, |
| "loss": 0.0002, |
| "step": 912 |
| }, |
| { |
| "clip_ratio": 0.0006421063735615462, |
| "completion_length": 126.6875, |
| "epoch": 0.1246586564718733, |
| "grad_norm": 0.0739646703004837, |
| "kl": 0.011297505276161246, |
| "learning_rate": 4.578947368421053e-07, |
| "loss": 0.0, |
| "num_tokens": 2683943.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.22499999403953552, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 913 |
| }, |
| { |
| "clip_ratio": 0.0012809407198801637, |
| "epoch": 0.12479519388312398, |
| "grad_norm": 0.07591162621974945, |
| "kl": 0.01175841357326135, |
| "learning_rate": 4.526315789473685e-07, |
| "loss": 0.0005, |
| "step": 914 |
| }, |
| { |
| "clip_ratio": 0.0007153579208534211, |
| "epoch": 0.12493173129437465, |
| "grad_norm": 0.07451826333999634, |
| "kl": 0.011770473574870266, |
| "learning_rate": 4.473684210526316e-07, |
| "loss": 0.0001, |
| "step": 915 |
| }, |
| { |
| "clip_ratio": 0.0011402182863093913, |
| "epoch": 0.12506826870562535, |
| "grad_norm": 0.07482127845287323, |
| "kl": 0.011633174843154848, |
| "learning_rate": 4.421052631578947e-07, |
| "loss": 0.0002, |
| "step": 916 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "completion_length": 109.46875, |
| "epoch": 0.125204806116876, |
| "grad_norm": 0.04029979184269905, |
| "kl": 0.011315665964502841, |
| "learning_rate": 4.3684210526315794e-07, |
| "loss": 0.0001, |
| "num_tokens": 2694806.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.6062500476837158, |
| "step": 917 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "epoch": 0.1253413435281267, |
| "grad_norm": 0.03924645110964775, |
| "kl": 0.0109497128505609, |
| "learning_rate": 4.3157894736842105e-07, |
| "loss": 0.0001, |
| "step": 918 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1254778809393774, |
| "grad_norm": 0.03933841362595558, |
| "kl": 0.01109032272506738, |
| "learning_rate": 4.2631578947368427e-07, |
| "loss": 0.0002, |
| "step": 919 |
| }, |
| { |
| "clip_ratio": 0.0002840909000951797, |
| "epoch": 0.12561441835062806, |
| "grad_norm": 0.03823743388056755, |
| "kl": 0.010918065345322248, |
| "learning_rate": 4.210526315789474e-07, |
| "loss": 0.0001, |
| "step": 920 |
| }, |
| { |
| "clip_ratio": 0.0012467819615267217, |
| "completion_length": 126.375, |
| "epoch": 0.12575095576187875, |
| "grad_norm": 0.044051431119441986, |
| "kl": 0.009680046583525836, |
| "learning_rate": 4.157894736842106e-07, |
| "loss": 0.0001, |
| "num_tokens": 2705934.0, |
| "reward": 0.71875, |
| "reward_std": 0.17745190858840942, |
| "rewards/reward_fn": 0.71875, |
| "step": 921 |
| }, |
| { |
| "clip_ratio": 0.000974522263277322, |
| "epoch": 0.12588749317312944, |
| "grad_norm": 0.049181364476680756, |
| "kl": 0.009984341231756844, |
| "learning_rate": 4.105263157894737e-07, |
| "loss": 0.0002, |
| "step": 922 |
| }, |
| { |
| "clip_ratio": 0.0005158415297046304, |
| "epoch": 0.1260240305843801, |
| "grad_norm": 0.042783595621585846, |
| "kl": 0.010040793160442263, |
| "learning_rate": 4.052631578947368e-07, |
| "loss": 0.0002, |
| "step": 923 |
| }, |
| { |
| "clip_ratio": 0.0010111908777616918, |
| "epoch": 0.1261605679956308, |
| "grad_norm": 0.056958772242069244, |
| "kl": 0.010024517934652977, |
| "learning_rate": 4.0000000000000003e-07, |
| "loss": 0.0001, |
| "step": 924 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 132.625, |
| "epoch": 0.1262971054068815, |
| "grad_norm": 0.03468235954642296, |
| "kl": 0.009052558852999937, |
| "learning_rate": 3.9473684210526315e-07, |
| "loss": 0.0001, |
| "num_tokens": 2717806.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 925 |
| }, |
| { |
| "clip_ratio": 0.00023148147738538682, |
| "epoch": 0.12643364281813216, |
| "grad_norm": 0.03451762720942497, |
| "kl": 0.009201876819133759, |
| "learning_rate": 3.8947368421052636e-07, |
| "loss": 0.0, |
| "step": 926 |
| }, |
| { |
| "clip_ratio": 0.00023148147738538682, |
| "epoch": 0.12657018022938285, |
| "grad_norm": 0.034427493810653687, |
| "kl": 0.009212747543642763, |
| "learning_rate": 3.8421052631578947e-07, |
| "loss": 0.0001, |
| "step": 927 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12670671764063354, |
| "grad_norm": 0.034289050847291946, |
| "kl": 0.008976701814390253, |
| "learning_rate": 3.789473684210527e-07, |
| "loss": 0.0, |
| "step": 928 |
| }, |
| { |
| "clip_ratio": 0.0006645138200838119, |
| "completion_length": 145.4375, |
| "epoch": 0.1268432550518842, |
| "grad_norm": 0.05551312118768692, |
| "kl": 0.01036488243698841, |
| "learning_rate": 3.736842105263158e-07, |
| "loss": 0.0003, |
| "num_tokens": 2729700.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 929 |
| }, |
| { |
| "clip_ratio": 0.0010838418529601768, |
| "epoch": 0.1269797924631349, |
| "grad_norm": 0.05485505238175392, |
| "kl": 0.010309238256013487, |
| "learning_rate": 3.6842105263157896e-07, |
| "loss": 0.0001, |
| "step": 930 |
| }, |
| { |
| "clip_ratio": 0.0006747905572410673, |
| "epoch": 0.1271163298743856, |
| "grad_norm": 0.05515707656741142, |
| "kl": 0.010451660557009745, |
| "learning_rate": 3.6315789473684213e-07, |
| "loss": 0.0001, |
| "step": 931 |
| }, |
| { |
| "clip_ratio": 0.0010947078408207744, |
| "epoch": 0.12725286728563626, |
| "grad_norm": 0.05624876916408539, |
| "kl": 0.01045336759852944, |
| "learning_rate": 3.578947368421053e-07, |
| "loss": 0.0001, |
| "step": 932 |
| }, |
| { |
| "clip_ratio": 0.0008043233392527327, |
| "completion_length": 128.5625, |
| "epoch": 0.12738940469688695, |
| "grad_norm": 0.06683102995157242, |
| "kl": 0.008829593745758757, |
| "learning_rate": 3.5263157894736846e-07, |
| "loss": 0.0001, |
| "num_tokens": 2741262.0, |
| "reward": 0.746874988079071, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.746874988079071, |
| "step": 933 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12752594210813764, |
| "grad_norm": 0.06684920191764832, |
| "kl": 0.00907902234030189, |
| "learning_rate": 3.4736842105263157e-07, |
| "loss": -0.0, |
| "step": 934 |
| }, |
| { |
| "clip_ratio": 0.0005558260454563424, |
| "epoch": 0.1276624795193883, |
| "grad_norm": 0.0667712539434433, |
| "kl": 0.009049147825862747, |
| "learning_rate": 3.421052631578948e-07, |
| "loss": -0.0001, |
| "step": 935 |
| }, |
| { |
| "clip_ratio": 0.000244140625, |
| "epoch": 0.127799016930639, |
| "grad_norm": 0.06718917191028595, |
| "kl": 0.008985225082142279, |
| "learning_rate": 3.368421052631579e-07, |
| "loss": -0.0002, |
| "step": 936 |
| }, |
| { |
| "clip_ratio": 0.0013269147311802953, |
| "completion_length": 141.78125, |
| "epoch": 0.1279355543418897, |
| "grad_norm": 0.08023970574140549, |
| "kl": 0.010123719010152854, |
| "learning_rate": 3.315789473684211e-07, |
| "loss": 0.0003, |
| "num_tokens": 2753059.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.4198557138442993, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 937 |
| }, |
| { |
| "clip_ratio": 0.00155613076640293, |
| "epoch": 0.12807209175314035, |
| "grad_norm": 0.07721517235040665, |
| "kl": 0.010227275008219294, |
| "learning_rate": 3.263157894736842e-07, |
| "loss": 0.0001, |
| "step": 938 |
| }, |
| { |
| "clip_ratio": 0.002012130498769693, |
| "epoch": 0.12820862916439105, |
| "grad_norm": 0.08134257048368454, |
| "kl": 0.010515461501199752, |
| "learning_rate": 3.210526315789474e-07, |
| "loss": 0.0001, |
| "step": 939 |
| }, |
| { |
| "clip_ratio": 0.0015320833190344274, |
| "epoch": 0.12834516657564174, |
| "grad_norm": 0.08687866479158401, |
| "kl": 0.010364998866862152, |
| "learning_rate": 3.1578947368421055e-07, |
| "loss": 0.0001, |
| "step": 940 |
| }, |
| { |
| "clip_ratio": 0.00048828125, |
| "completion_length": 120.28125, |
| "epoch": 0.1284817039868924, |
| "grad_norm": 0.0412849485874176, |
| "kl": 0.012160152327851392, |
| "learning_rate": 3.105263157894737e-07, |
| "loss": 0.0001, |
| "num_tokens": 2764020.0, |
| "reward": 0.8875000476837158, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.8875000476837158, |
| "step": 941 |
| }, |
| { |
| "clip_ratio": 0.0007553752802778035, |
| "epoch": 0.1286182413981431, |
| "grad_norm": 0.04290853440761566, |
| "kl": 0.012012018290988635, |
| "learning_rate": 3.052631578947369e-07, |
| "loss": 0.0001, |
| "step": 942 |
| }, |
| { |
| "clip_ratio": 0.00048828125, |
| "epoch": 0.12875477880939376, |
| "grad_norm": 0.041789595037698746, |
| "kl": 0.011813177145086229, |
| "learning_rate": 3.0000000000000004e-07, |
| "loss": 0.0, |
| "step": 943 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.12889131622064445, |
| "grad_norm": 0.041725724935531616, |
| "kl": 0.011911887653695885, |
| "learning_rate": 2.9473684210526315e-07, |
| "loss": 0.0001, |
| "step": 944 |
| }, |
| { |
| "clip_ratio": 0.00128199890605174, |
| "completion_length": 100.21875, |
| "epoch": 0.12902785363189515, |
| "grad_norm": 0.042603928595781326, |
| "kl": 0.012103870598366484, |
| "learning_rate": 2.894736842105263e-07, |
| "loss": 0.0003, |
| "num_tokens": 2774091.0, |
| "reward": 0.550000011920929, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.550000011920929, |
| "step": 945 |
| }, |
| { |
| "clip_ratio": 0.0009143518400378525, |
| "epoch": 0.1291643910431458, |
| "grad_norm": 0.043181583285331726, |
| "kl": 0.0123589242430171, |
| "learning_rate": 2.842105263157895e-07, |
| "loss": 0.0001, |
| "step": 946 |
| }, |
| { |
| "clip_ratio": 0.0007592766778543591, |
| "epoch": 0.1293009284543965, |
| "grad_norm": 0.04348016530275345, |
| "kl": 0.012284232841921039, |
| "learning_rate": 2.7894736842105264e-07, |
| "loss": 0.0002, |
| "step": 947 |
| }, |
| { |
| "clip_ratio": 0.00128199890605174, |
| "epoch": 0.1294374658656472, |
| "grad_norm": 0.04269959777593613, |
| "kl": 0.012165894513600506, |
| "learning_rate": 2.736842105263158e-07, |
| "loss": 0.0002, |
| "step": 948 |
| }, |
| { |
| "clip_ratio": 0.00012400794366840273, |
| "completion_length": 125.59375, |
| "epoch": 0.12957400327689786, |
| "grad_norm": 0.033528104424476624, |
| "kl": 0.010424962820252404, |
| "learning_rate": 2.6842105263157897e-07, |
| "loss": -0.0, |
| "num_tokens": 2785454.0, |
| "reward": 0.8875000476837158, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.8875000476837158, |
| "step": 949 |
| }, |
| { |
| "clip_ratio": 0.00022644927958026528, |
| "epoch": 0.12971054068814855, |
| "grad_norm": 0.03530941903591156, |
| "kl": 0.01039945226511918, |
| "learning_rate": 2.6315789473684213e-07, |
| "loss": 0.0002, |
| "step": 950 |
| }, |
| { |
| "clip_ratio": 0.0002456032743793912, |
| "epoch": 0.12984707809939924, |
| "grad_norm": 0.03371855244040489, |
| "kl": 0.010452555812662467, |
| "learning_rate": 2.578947368421053e-07, |
| "loss": 0.0001, |
| "step": 951 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1299836155106499, |
| "grad_norm": 0.035171639174222946, |
| "kl": 0.010235989175271243, |
| "learning_rate": 2.5263157894736846e-07, |
| "loss": 0.0001, |
| "step": 952 |
| }, |
| { |
| "clip_ratio": 0.0004131752939429134, |
| "completion_length": 120.65625, |
| "epoch": 0.1301201529219006, |
| "grad_norm": 0.06675706058740616, |
| "kl": 0.010402317508123815, |
| "learning_rate": 2.473684210526316e-07, |
| "loss": 0.0002, |
| "num_tokens": 2796687.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.31605762243270874, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 953 |
| }, |
| { |
| "clip_ratio": 0.00038941160892136395, |
| "epoch": 0.1302566903331513, |
| "grad_norm": 0.06641741096973419, |
| "kl": 0.010149119298148435, |
| "learning_rate": 2.4210526315789473e-07, |
| "loss": 0.0001, |
| "step": 954 |
| }, |
| { |
| "clip_ratio": 0.00047757322317920625, |
| "epoch": 0.13039322774440196, |
| "grad_norm": 0.06666306406259537, |
| "kl": 0.010389204719103873, |
| "learning_rate": 2.3684210526315792e-07, |
| "loss": 0.0001, |
| "step": 955 |
| }, |
| { |
| "clip_ratio": 0.0007592200126964599, |
| "epoch": 0.13052976515565265, |
| "grad_norm": 0.06657683104276657, |
| "kl": 0.010527671533054672, |
| "learning_rate": 2.315789473684211e-07, |
| "loss": -0.0001, |
| "step": 956 |
| }, |
| { |
| "clip_ratio": 0.0005896751245018095, |
| "completion_length": 120.03125, |
| "epoch": 0.13066630256690334, |
| "grad_norm": 0.06373964250087738, |
| "kl": 0.010264166303386446, |
| "learning_rate": 2.2631578947368425e-07, |
| "loss": 0.0001, |
| "num_tokens": 2807348.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.2337019145488739, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 957 |
| }, |
| { |
| "clip_ratio": 0.0005132435617269948, |
| "epoch": 0.130802839978154, |
| "grad_norm": 0.06462369114160538, |
| "kl": 0.010361698499764316, |
| "learning_rate": 2.2105263157894736e-07, |
| "loss": 0.0001, |
| "step": 958 |
| }, |
| { |
| "clip_ratio": 0.0005185053450986743, |
| "epoch": 0.1309393773894047, |
| "grad_norm": 0.06394156813621521, |
| "kl": 0.01056599643197842, |
| "learning_rate": 2.1578947368421053e-07, |
| "loss": -0.0001, |
| "step": 959 |
| }, |
| { |
| "clip_ratio": 0.000292056065518409, |
| "epoch": 0.1310759148006554, |
| "grad_norm": 0.06401284784078598, |
| "kl": 0.010314612052752636, |
| "learning_rate": 2.105263157894737e-07, |
| "loss": 0.0001, |
| "step": 960 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.875, |
| "epoch": 0.13121245221190606, |
| "grad_norm": 0.023246673867106438, |
| "kl": 0.010112395939358976, |
| "learning_rate": 2.0526315789473685e-07, |
| "loss": 0.0001, |
| "num_tokens": 2818076.0, |
| "reward": 0.859375, |
| "reward_std": 0.05624999850988388, |
| "rewards/reward_fn": 0.859375, |
| "step": 961 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.13134898962315675, |
| "grad_norm": 0.023142091929912567, |
| "kl": 0.010110046772751957, |
| "learning_rate": 2.0000000000000002e-07, |
| "loss": 0.0, |
| "step": 962 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.13148552703440744, |
| "grad_norm": 0.023380635306239128, |
| "kl": 0.010197044037340675, |
| "learning_rate": 1.9473684210526318e-07, |
| "loss": 0.0001, |
| "step": 963 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1316220644456581, |
| "grad_norm": 0.02310342527925968, |
| "kl": 0.009969949518563226, |
| "learning_rate": 1.8947368421052634e-07, |
| "loss": 0.0, |
| "step": 964 |
| }, |
| { |
| "clip_ratio": 0.0008272800769191235, |
| "completion_length": 127.90625, |
| "epoch": 0.1317586018569088, |
| "grad_norm": 0.06340518593788147, |
| "kl": 0.012789289277861826, |
| "learning_rate": 1.8421052631578948e-07, |
| "loss": 0.0001, |
| "num_tokens": 2829673.0, |
| "reward": 0.606249988079071, |
| "reward_std": 0.2424038052558899, |
| "rewards/reward_fn": 0.606249988079071, |
| "step": 965 |
| }, |
| { |
| "clip_ratio": 0.001213868526974693, |
| "epoch": 0.1318951392681595, |
| "grad_norm": 0.05466872826218605, |
| "kl": 0.012126193032599986, |
| "learning_rate": 1.7894736842105265e-07, |
| "loss": 0.0001, |
| "step": 966 |
| }, |
| { |
| "clip_ratio": 0.00022482014901470393, |
| "epoch": 0.13203167667941015, |
| "grad_norm": 0.05456630140542984, |
| "kl": 0.012076701867044903, |
| "learning_rate": 1.7368421052631578e-07, |
| "loss": 0.0002, |
| "step": 967 |
| }, |
| { |
| "clip_ratio": 0.0008625764457974583, |
| "epoch": 0.13216821409066085, |
| "grad_norm": 0.061798643320798874, |
| "kl": 0.01273639798455406, |
| "learning_rate": 1.6842105263157895e-07, |
| "loss": 0.0002, |
| "step": 968 |
| }, |
| { |
| "clip_ratio": 0.001317727321293205, |
| "completion_length": 120.9375, |
| "epoch": 0.1323047515019115, |
| "grad_norm": 0.051433056592941284, |
| "kl": 0.010656341539288405, |
| "learning_rate": 1.631578947368421e-07, |
| "loss": 0.0003, |
| "num_tokens": 2840855.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.18615379929542542, |
| "rewards/reward_fn": 0.8031250238418579, |
| "step": 969 |
| }, |
| { |
| "clip_ratio": 0.0011772566940635443, |
| "epoch": 0.1324412889131622, |
| "grad_norm": 0.051887210458517075, |
| "kl": 0.010402877880551387, |
| "learning_rate": 1.5789473684210527e-07, |
| "loss": 0.0003, |
| "step": 970 |
| }, |
| { |
| "clip_ratio": 0.0005079461698187515, |
| "epoch": 0.1325778263244129, |
| "grad_norm": 0.052023597061634064, |
| "kl": 0.010541295756411273, |
| "learning_rate": 1.5263157894736844e-07, |
| "loss": 0.0003, |
| "step": 971 |
| }, |
| { |
| "clip_ratio": 0.00045540845894720405, |
| "epoch": 0.13271436373566356, |
| "grad_norm": 0.05091957375407219, |
| "kl": 0.010208568717644084, |
| "learning_rate": 1.4736842105263158e-07, |
| "loss": -0.0, |
| "step": 972 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.375, |
| "epoch": 0.13285090114691425, |
| "grad_norm": 0.03053058311343193, |
| "kl": 0.009520908235572278, |
| "learning_rate": 1.4210526315789474e-07, |
| "loss": 0.0002, |
| "num_tokens": 2851411.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 973 |
| }, |
| { |
| "clip_ratio": 0.0003324467979837209, |
| "epoch": 0.13298743855816494, |
| "grad_norm": 0.030359813943505287, |
| "kl": 0.009604914383089636, |
| "learning_rate": 1.368421052631579e-07, |
| "loss": 0.0, |
| "step": 974 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1331239759694156, |
| "grad_norm": 0.03019576333463192, |
| "kl": 0.009532538024359383, |
| "learning_rate": 1.3157894736842107e-07, |
| "loss": 0.0001, |
| "step": 975 |
| }, |
| { |
| "clip_ratio": 0.0006648935959674418, |
| "epoch": 0.1332605133806663, |
| "grad_norm": 0.029987657442688942, |
| "kl": 0.009747071948368102, |
| "learning_rate": 1.2631578947368423e-07, |
| "loss": 0.0001, |
| "step": 976 |
| }, |
| { |
| "clip_ratio": 0.0008955505472840741, |
| "completion_length": 115.21875, |
| "epoch": 0.133397050791917, |
| "grad_norm": 0.041872940957546234, |
| "kl": 0.011071445827838033, |
| "learning_rate": 1.2105263157894737e-07, |
| "loss": 0.0002, |
| "num_tokens": 2862118.0, |
| "reward": 0.7749999761581421, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.7749999761581421, |
| "step": 977 |
| }, |
| { |
| "clip_ratio": 0.00048095356032717973, |
| "epoch": 0.13353358820316766, |
| "grad_norm": 0.04186861962080002, |
| "kl": 0.01133234730514232, |
| "learning_rate": 1.1578947368421054e-07, |
| "loss": 0.0002, |
| "step": 978 |
| }, |
| { |
| "clip_ratio": 0.0017156863468699157, |
| "epoch": 0.13367012561441835, |
| "grad_norm": 0.042711351066827774, |
| "kl": 0.011379848656360991, |
| "learning_rate": 1.1052631578947368e-07, |
| "loss": 0.0002, |
| "step": 979 |
| }, |
| { |
| "clip_ratio": 0.00017458100046496838, |
| "epoch": 0.13380666302566904, |
| "grad_norm": 0.04217659309506416, |
| "kl": 0.011275138182099909, |
| "learning_rate": 1.0526315789473685e-07, |
| "loss": 0.0002, |
| "step": 980 |
| }, |
| { |
| "clip_ratio": 0.000996959744952619, |
| "completion_length": 117.15625, |
| "epoch": 0.1339432004369197, |
| "grad_norm": 0.07359009236097336, |
| "kl": 0.011579404235817492, |
| "learning_rate": 1.0000000000000001e-07, |
| "loss": 0.0004, |
| "num_tokens": 2872843.0, |
| "reward": 0.859375, |
| "reward_std": 0.2337018996477127, |
| "rewards/reward_fn": 0.859375, |
| "step": 981 |
| }, |
| { |
| "clip_ratio": 0.0011051351903006434, |
| "epoch": 0.1340797378481704, |
| "grad_norm": 0.07498800754547119, |
| "kl": 0.011332578738802113, |
| "learning_rate": 9.473684210526317e-08, |
| "loss": 0.0004, |
| "step": 982 |
| }, |
| { |
| "clip_ratio": 0.0006770530017092824, |
| "epoch": 0.1342162752594211, |
| "grad_norm": 0.07397735118865967, |
| "kl": 0.011369323045073543, |
| "learning_rate": 8.947368421052632e-08, |
| "loss": 0.0003, |
| "step": 983 |
| }, |
| { |
| "clip_ratio": 0.00042808218859136105, |
| "epoch": 0.13435281267067176, |
| "grad_norm": 0.07387201488018036, |
| "kl": 0.011691686740959994, |
| "learning_rate": 8.421052631578947e-08, |
| "loss": 0.0004, |
| "step": 984 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.46875, |
| "epoch": 0.13448935008192245, |
| "grad_norm": 0.035393666476011276, |
| "kl": 0.012935981023474596, |
| "learning_rate": 7.894736842105264e-08, |
| "loss": 0.0001, |
| "num_tokens": 2883250.0, |
| "reward": 0.8312499523162842, |
| "reward_std": 0.06495190411806107, |
| "rewards/reward_fn": 0.8312499523162842, |
| "step": 985 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.13462588749317314, |
| "grad_norm": 0.03454882651567459, |
| "kl": 0.013269755116198212, |
| "learning_rate": 7.368421052631579e-08, |
| "loss": 0.0001, |
| "step": 986 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1347624249044238, |
| "grad_norm": 0.03491717576980591, |
| "kl": 0.013173356375773437, |
| "learning_rate": 6.842105263157895e-08, |
| "loss": 0.0001, |
| "step": 987 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1348989623156745, |
| "grad_norm": 0.034808382391929626, |
| "kl": 0.01312936055182945, |
| "learning_rate": 6.315789473684211e-08, |
| "loss": 0.0002, |
| "step": 988 |
| }, |
| { |
| "clip_ratio": 0.0011555317978491075, |
| "completion_length": 145.53125, |
| "epoch": 0.1350354997269252, |
| "grad_norm": 0.07175701856613159, |
| "kl": 0.010380461098975502, |
| "learning_rate": 5.789473684210527e-08, |
| "loss": 0.0005, |
| "num_tokens": 2895079.0, |
| "reward": 0.6343749761581421, |
| "reward_std": 0.298653781414032, |
| "rewards/reward_fn": 0.6343749761581421, |
| "step": 989 |
| }, |
| { |
| "clip_ratio": 0.0003294239650131203, |
| "epoch": 0.13517203713817585, |
| "grad_norm": 0.06990694254636765, |
| "kl": 0.010374384561146144, |
| "learning_rate": 5.263157894736842e-08, |
| "loss": 0.0003, |
| "step": 990 |
| }, |
| { |
| "clip_ratio": 0.0007133064937079325, |
| "epoch": 0.13530857454942655, |
| "grad_norm": 0.07134159654378891, |
| "kl": 0.01048758153046947, |
| "learning_rate": 4.7368421052631586e-08, |
| "loss": 0.0001, |
| "step": 991 |
| }, |
| { |
| "clip_ratio": 0.0006030536605976522, |
| "epoch": 0.13544511196067724, |
| "grad_norm": 0.07197128981351852, |
| "kl": 0.01038069138303399, |
| "learning_rate": 4.2105263157894737e-08, |
| "loss": 0.0003, |
| "step": 992 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 127.6875, |
| "epoch": 0.1355816493719279, |
| "grad_norm": 0.06501730531454086, |
| "kl": 0.011816246449598111, |
| "learning_rate": 3.6842105263157894e-08, |
| "loss": -0.0001, |
| "num_tokens": 2906801.0, |
| "reward": 0.7468750476837158, |
| "reward_std": 0.25110572576522827, |
| "rewards/reward_fn": 0.7468750476837158, |
| "step": 993 |
| }, |
| { |
| "clip_ratio": 0.0011552878568181768, |
| "epoch": 0.1357181867831786, |
| "grad_norm": 0.06590738892555237, |
| "kl": 0.012002686664345674, |
| "learning_rate": 3.157894736842106e-08, |
| "loss": 0.0, |
| "step": 994 |
| }, |
| { |
| "clip_ratio": 0.00042808218859136105, |
| "epoch": 0.13585472419442926, |
| "grad_norm": 0.064786896109581, |
| "kl": 0.011948928302444983, |
| "learning_rate": 2.631578947368421e-08, |
| "loss": 0.0002, |
| "step": 995 |
| }, |
| { |
| "clip_ratio": 0.0013783742906525731, |
| "epoch": 0.13599126160567995, |
| "grad_norm": 0.06537439674139023, |
| "kl": 0.011855509001179598, |
| "learning_rate": 2.1052631578947368e-08, |
| "loss": 0.0, |
| "step": 996 |
| }, |
| { |
| "clip_ratio": 0.0005120444984640926, |
| "completion_length": 116.15625, |
| "epoch": 0.13612779901693065, |
| "grad_norm": 0.03533847630023956, |
| "kl": 0.009305741070420481, |
| "learning_rate": 1.578947368421053e-08, |
| "loss": 0.0001, |
| "num_tokens": 2917850.0, |
| "reward": 0.71875, |
| "reward_std": 0.11249999701976776, |
| "rewards/reward_fn": 0.71875, |
| "step": 997 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.1362643364281813, |
| "grad_norm": 0.03546489402651787, |
| "kl": 0.009112709522014484, |
| "learning_rate": 1.0526315789473684e-08, |
| "loss": -0.0001, |
| "step": 998 |
| }, |
| { |
| "clip_ratio": 0.00017959770048037171, |
| "epoch": 0.136400873839432, |
| "grad_norm": 0.03534821793437004, |
| "kl": 0.009316969342762604, |
| "learning_rate": 5.263157894736842e-09, |
| "loss": 0.0001, |
| "step": 999 |
| }, |
| { |
| "clip_ratio": 0.00017959770048037171, |
| "epoch": 0.1365374112506827, |
| "grad_norm": 0.03498006984591484, |
| "kl": 0.009425293108506594, |
| "learning_rate": 0.0, |
| "loss": 0.0001, |
| "step": 1000 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 1000, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 250, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 32, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
