{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.021367521367521368, "eval_steps": 500, "global_step": 50, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5555555820465088, "completions/max_length": 3000.0, "completions/max_terminated_length": 2837.0, "completions/mean_length": 2626.97216796875, "completions/mean_terminated_length": 2160.6875, "completions/min_length": 1285.0, "completions/min_terminated_length": 1285.0, "entropy": 0.17718915144602457, "epoch": 0.00042735042735042735, "frac_reward_zero_std": 0.0, "grad_norm": 0.07994551956653595, "learning_rate": 0.0, "loss": 0.0468, "num_tokens": 102485.0, "reward": -1.153435230255127, "reward_std": 1.0195810794830322, "rewards/hint_following/mean": -0.2777777910232544, "rewards/hint_following/std": 0.8819171190261841, "rewards/length_penalty/mean": -0.8756573796272278, "rewards/length_penalty/std": 0.17644810676574707, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9937517642974854, "sampling/importance_sampling_ratio/min": 0.16920268535614014, "sampling/sampling_logp_difference/max": 1.7766579389572144, "sampling/sampling_logp_difference/mean": 0.02078823931515217, "step": 1, "step_time": 67.38753714214545 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 3000.0, "completions/max_terminated_length": 2961.0, "completions/mean_length": 1809.4444580078125, "completions/mean_terminated_length": 1412.5926513671875, "completions/min_length": 858.0, "completions/min_terminated_length": 858.0, "entropy": 0.1950028215845426, "epoch": 0.0008547008547008547, "frac_reward_zero_std": 0.0, "grad_norm": 0.07918090373277664, "learning_rate": 3.846153846153847e-06, "loss": 0.022, "num_tokens": 177279.0, "reward": -0.4642592668533325, "reward_std": 1.043834924697876, "rewards/hint_following/mean": 0.1388888955116272, "rewards/hint_following/std": 0.798311710357666, "rewards/length_penalty/mean": -0.6031482219696045, "rewards/length_penalty/std": 0.28368300199508667, "sampling/importance_sampling_ratio/max": 2.0890045166015625, "sampling/importance_sampling_ratio/mean": 0.9933527708053589, "sampling/importance_sampling_ratio/min": 0.29880645871162415, "sampling/sampling_logp_difference/max": 1.2079591751098633, "sampling/sampling_logp_difference/mean": 0.021868562325835228, "step": 2, "step_time": 59.25448451703414 }, { "clip_ratio/high_max": 0.007054112308348219, "clip_ratio/high_mean": 0.007054112308348219, "clip_ratio/low_mean": 0.0024281112127937376, "clip_ratio/low_min": 0.0024281112127937376, "clip_ratio/region_mean": 0.009482223695764938, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 3000.0, "completions/max_terminated_length": 2411.0, "completions/mean_length": 1750.8055419921875, "completions/mean_terminated_length": 1500.966796875, "completions/min_length": 939.0, "completions/min_terminated_length": 939.0, "entropy": 0.1983068659901619, "epoch": 0.001282051282051282, "frac_reward_zero_std": 0.0, "grad_norm": 0.06695634126663208, "learning_rate": 7.692307692307694e-06, "loss": 0.0035, "num_tokens": 245288.0, "reward": -0.25026851892471313, "reward_std": 0.9555131196975708, "rewards/hint_following/mean": 0.3333333432674408, "rewards/hint_following/std": 0.7559289932250977, "rewards/length_penalty/mean": -0.5836018323898315, "rewards/length_penalty/std": 0.22866536676883698, "sampling/importance_sampling_ratio/max": 2.208913803100586, "sampling/importance_sampling_ratio/mean": 0.9931869506835938, "sampling/importance_sampling_ratio/min": 0.30834895372390747, "sampling/sampling_logp_difference/max": 1.176523208618164, "sampling/sampling_logp_difference/mean": 0.022443262860178947, "step": 3, "step_time": 55.47524960897863 }, { "clip_ratio/high_max": 0.0025329499427850046, "clip_ratio/high_mean": 0.0025329499427850046, "clip_ratio/low_mean": 0.0070530143566429615, "clip_ratio/low_min": 0.0070530143566429615, "clip_ratio/region_mean": 0.009585964183012644, "completions/clipped_ratio": 0.472222238779068, "completions/max_length": 3000.0, "completions/max_terminated_length": 2857.0, "completions/mean_length": 2469.27783203125, "completions/mean_terminated_length": 1994.4210205078125, "completions/min_length": 627.0, "completions/min_terminated_length": 627.0, "entropy": 0.1949674834807714, "epoch": 0.0017094017094017094, "frac_reward_zero_std": 0.0, "grad_norm": 0.08172055333852768, "learning_rate": 1.153846153846154e-05, "loss": 0.0355, "num_tokens": 339768.0, "reward": -1.0453147888183594, "reward_std": 0.8787956237792969, "rewards/hint_following/mean": -0.2222222238779068, "rewards/hint_following/std": 0.7601169347763062, "rewards/length_penalty/mean": -0.8230926394462585, "rewards/length_penalty/std": 0.2654260993003845, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.993114173412323, "sampling/importance_sampling_ratio/min": 0.19437530636787415, "sampling/sampling_logp_difference/max": 1.6379644870758057, "sampling/sampling_logp_difference/mean": 0.022510718554258347, "step": 4, "step_time": 62.77226335310843 }, { "clip_ratio/high_max": 0.001851275311006854, "clip_ratio/high_mean": 0.001851275311006854, "clip_ratio/low_mean": 0.00705820966201524, "clip_ratio/low_min": 0.00705820966201524, "clip_ratio/region_mean": 0.008909484914814433, "completions/clipped_ratio": 0.4166666567325592, "completions/max_length": 3000.0, "completions/max_terminated_length": 2915.0, "completions/mean_length": 2362.8056640625, "completions/mean_terminated_length": 1907.666748046875, "completions/min_length": 1109.0, "completions/min_terminated_length": 1109.0, "entropy": 0.18256587783495584, "epoch": 0.002136752136752137, "frac_reward_zero_std": 0.0, "grad_norm": 0.072978176176548, "learning_rate": 1.5384615384615387e-05, "loss": 0.1055, "num_tokens": 431687.0, "reward": -0.9264907836914062, "reward_std": 1.0278923511505127, "rewards/hint_following/mean": -0.1388888955116272, "rewards/hint_following/std": 0.8333333730697632, "rewards/length_penalty/mean": -0.787601888179779, "rewards/length_penalty/std": 0.2262839674949646, "sampling/importance_sampling_ratio/max": 2.5575737953186035, "sampling/importance_sampling_ratio/mean": 0.9932507872581482, "sampling/importance_sampling_ratio/min": 0.2952674627304077, "sampling/sampling_logp_difference/max": 1.2198736667633057, "sampling/sampling_logp_difference/mean": 0.020803121849894524, "step": 5, "step_time": 61.06088067602832 }, { "clip_ratio/high_max": 0.0030960943549871445, "clip_ratio/high_mean": 0.0030960943549871445, "clip_ratio/low_mean": 0.006137795591106017, "clip_ratio/low_min": 0.006137795591106017, "clip_ratio/region_mean": 0.009233890101313591, "completions/clipped_ratio": 0.4166666567325592, "completions/max_length": 3000.0, "completions/max_terminated_length": 2929.0, "completions/mean_length": 2417.444580078125, "completions/mean_terminated_length": 2001.3333740234375, "completions/min_length": 1434.0, "completions/min_terminated_length": 1434.0, "entropy": 0.1919809157649676, "epoch": 0.002564102564102564, "frac_reward_zero_std": 0.0, "grad_norm": 0.07703683525323868, "learning_rate": 1.923076923076923e-05, "loss": 0.1019, "num_tokens": 525555.0, "reward": -0.7224815487861633, "reward_std": 1.1388682126998901, "rewards/hint_following/mean": 0.0833333358168602, "rewards/hint_following/std": 0.9673232436180115, "rewards/length_penalty/mean": -0.8058148622512817, "rewards/length_penalty/std": 0.18899519741535187, "sampling/importance_sampling_ratio/max": 2.9181816577911377, "sampling/importance_sampling_ratio/mean": 0.9935172200202942, "sampling/importance_sampling_ratio/min": 0.14165793359279633, "sampling/sampling_logp_difference/max": 1.9543399810791016, "sampling/sampling_logp_difference/mean": 0.021658122539520264, "step": 6, "step_time": 64.93633897381369 }, { "clip_ratio/high_max": 0.004218692930104832, "clip_ratio/high_mean": 0.004218692930104832, "clip_ratio/low_mean": 0.0048820926652600365, "clip_ratio/low_min": 0.0048820926652600365, "clip_ratio/region_mean": 0.009100785556559762, "completions/clipped_ratio": 0.2222222238779068, "completions/max_length": 3000.0, "completions/max_terminated_length": 2933.0, "completions/mean_length": 2029.0833740234375, "completions/mean_terminated_length": 1751.6785888671875, "completions/min_length": 1009.0, "completions/min_terminated_length": 1009.0, "entropy": 0.20896865924199423, "epoch": 0.0029914529914529917, "frac_reward_zero_std": 0.0, "grad_norm": 0.07187572866678238, "learning_rate": 2.307692307692308e-05, "loss": 0.0798, "num_tokens": 606186.0, "reward": -0.4819166362285614, "reward_std": 0.9970327019691467, "rewards/hint_following/mean": 0.1944444477558136, "rewards/hint_following/std": 0.7862913012504578, "rewards/length_penalty/mean": -0.676361083984375, "rewards/length_penalty/std": 0.2366827428340912, "sampling/importance_sampling_ratio/max": 2.6964023113250732, "sampling/importance_sampling_ratio/mean": 0.9927431344985962, "sampling/importance_sampling_ratio/min": 0.21346372365951538, "sampling/sampling_logp_difference/max": 1.5442883968353271, "sampling/sampling_logp_difference/mean": 0.022996528074145317, "step": 7, "step_time": 59.43198294797912 }, { "clip_ratio/high_max": 0.007654782539854447, "clip_ratio/high_mean": 0.007654782539854447, "clip_ratio/low_mean": 0.002519401100774606, "clip_ratio/low_min": 0.002519401100774606, "clip_ratio/region_mean": 0.010174183640629053, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 3000.0, "completions/max_terminated_length": 2957.0, "completions/mean_length": 2227.97216796875, "completions/mean_terminated_length": 2073.56689453125, "completions/min_length": 1197.0, "completions/min_terminated_length": 1197.0, "entropy": 0.19751630226771036, "epoch": 0.003418803418803419, "frac_reward_zero_std": 0.0, "grad_norm": 0.08499564230442047, "learning_rate": 2.6923076923076923e-05, "loss": 0.0572, "num_tokens": 692279.0, "reward": -0.4093240797519684, "reward_std": 0.8091796040534973, "rewards/hint_following/mean": 0.3333333432674408, "rewards/hint_following/std": 0.7171371579170227, "rewards/length_penalty/mean": -0.7426574230194092, "rewards/length_penalty/std": 0.19112928211688995, "sampling/importance_sampling_ratio/max": 2.491806745529175, "sampling/importance_sampling_ratio/mean": 0.9929631352424622, "sampling/importance_sampling_ratio/min": 0.3287244141101837, "sampling/sampling_logp_difference/max": 1.1125354766845703, "sampling/sampling_logp_difference/mean": 0.021952621638774872, "step": 8, "step_time": 60.087568536167964 }, { "clip_ratio/high_max": 0.007754642438764374, "clip_ratio/high_mean": 0.007754642438764374, "clip_ratio/low_mean": 0.0032427439582534134, "clip_ratio/low_min": 0.0032427439582534134, "clip_ratio/region_mean": 0.010997386649250984, "completions/clipped_ratio": 0.1111111119389534, "completions/max_length": 3000.0, "completions/max_terminated_length": 2948.0, "completions/mean_length": 2034.2222900390625, "completions/mean_terminated_length": 1913.5, "completions/min_length": 778.0, "completions/min_terminated_length": 778.0, "entropy": 0.21248381833235422, "epoch": 0.0038461538461538464, "frac_reward_zero_std": 0.0, "grad_norm": 0.07896988093852997, "learning_rate": 3.0769230769230774e-05, "loss": 0.0952, "num_tokens": 771241.0, "reward": -0.28918519616127014, "reward_std": 0.7873722314834595, "rewards/hint_following/mean": 0.3888888955116272, "rewards/hint_following/std": 0.6877615451812744, "rewards/length_penalty/mean": -0.6780741214752197, "rewards/length_penalty/std": 0.22194787859916687, "sampling/importance_sampling_ratio/max": 2.757654905319214, "sampling/importance_sampling_ratio/mean": 0.9926807880401611, "sampling/importance_sampling_ratio/min": 0.19047939777374268, "sampling/sampling_logp_difference/max": 1.6582112312316895, "sampling/sampling_logp_difference/mean": 0.02367428131401539, "step": 9, "step_time": 58.2687593010487 }, { "clip_ratio/high_max": 0.0022279491822700948, "clip_ratio/high_mean": 0.0022279491822700948, "clip_ratio/low_mean": 0.0019324645400047302, "clip_ratio/low_min": 0.0019324645400047302, "clip_ratio/region_mean": 0.004160413751378655, "completions/clipped_ratio": 0.3611111044883728, "completions/max_length": 3000.0, "completions/max_terminated_length": 2960.0, "completions/mean_length": 2266.111083984375, "completions/mean_terminated_length": 1851.3043212890625, "completions/min_length": 784.0, "completions/min_terminated_length": 784.0, "entropy": 0.20578551292419434, "epoch": 0.004273504273504274, "frac_reward_zero_std": 0.0, "grad_norm": 0.06615690886974335, "learning_rate": 3.461538461538462e-05, "loss": 0.0543, "num_tokens": 858521.0, "reward": -0.5331481099128723, "reward_std": 1.1243869066238403, "rewards/hint_following/mean": 0.2222222238779068, "rewards/hint_following/std": 0.929242730140686, "rewards/length_penalty/mean": -0.7553703784942627, "rewards/length_penalty/std": 0.23448802530765533, "sampling/importance_sampling_ratio/max": 2.0015952587127686, "sampling/importance_sampling_ratio/mean": 0.9925298094749451, "sampling/importance_sampling_ratio/min": 0.30108708143234253, "sampling/sampling_logp_difference/max": 1.2003557682037354, "sampling/sampling_logp_difference/mean": 0.021938424557447433, "step": 10, "step_time": 60.113012868212536 }, { "clip_ratio/high_max": 0.002628828883947184, "clip_ratio/high_mean": 0.002628828883947184, "clip_ratio/low_mean": 0.0007913654941755036, "clip_ratio/low_min": 0.0007913654941755036, "clip_ratio/region_mean": 0.0034201944169277945, "completions/clipped_ratio": 0.3888888955116272, "completions/max_length": 3000.0, "completions/max_terminated_length": 2856.0, "completions/mean_length": 2308.111083984375, "completions/mean_terminated_length": 1867.8182373046875, "completions/min_length": 853.0, "completions/min_terminated_length": 853.0, "entropy": 0.1780667081475258, "epoch": 0.004700854700854701, "frac_reward_zero_std": 0.0, "grad_norm": 0.06938029080629349, "learning_rate": 3.846153846153846e-05, "loss": 0.0506, "num_tokens": 948171.0, "reward": -1.0471482276916504, "reward_std": 0.824184238910675, "rewards/hint_following/mean": -0.2777777910232544, "rewards/hint_following/std": 0.6594851613044739, "rewards/length_penalty/mean": -0.7693703770637512, "rewards/length_penalty/std": 0.24463628232479095, "sampling/importance_sampling_ratio/max": 2.5170931816101074, "sampling/importance_sampling_ratio/mean": 0.9934613108634949, "sampling/importance_sampling_ratio/min": 0.29526859521865845, "sampling/sampling_logp_difference/max": 1.21986985206604, "sampling/sampling_logp_difference/mean": 0.021038096398115158, "step": 11, "step_time": 61.20216279185843 }, { "clip_ratio/high_max": 0.009717889750997225, "clip_ratio/high_mean": 0.009717889750997225, "clip_ratio/low_mean": 0.001428690991209199, "clip_ratio/low_min": 0.001428690991209199, "clip_ratio/region_mean": 0.011146580955634514, "completions/clipped_ratio": 0.2777777910232544, "completions/max_length": 3000.0, "completions/max_terminated_length": 2997.0, "completions/mean_length": 2494.944580078125, "completions/mean_terminated_length": 2300.6923828125, "completions/min_length": 1195.0, "completions/min_terminated_length": 1195.0, "entropy": 0.21741948276758194, "epoch": 0.005128205128205128, "frac_reward_zero_std": 0.0, "grad_norm": 0.08027122914791107, "learning_rate": 4.230769230769231e-05, "loss": 0.0271, "num_tokens": 1045165.0, "reward": -0.6094259023666382, "reward_std": 0.9179477691650391, "rewards/hint_following/mean": 0.2222222238779068, "rewards/hint_following/std": 0.8655670881271362, "rewards/length_penalty/mean": -0.8316481113433838, "rewards/length_penalty/std": 0.1910414844751358, "sampling/importance_sampling_ratio/max": 2.7227704524993896, "sampling/importance_sampling_ratio/mean": 0.9923471808433533, "sampling/importance_sampling_ratio/min": 0.309160441160202, "sampling/sampling_logp_difference/max": 1.1738948822021484, "sampling/sampling_logp_difference/mean": 0.02328832633793354, "step": 12, "step_time": 63.202891704044305 }, { "clip_ratio/high_max": 0.0020594243348265686, "clip_ratio/high_mean": 0.0020594243348265686, "clip_ratio/low_mean": 0.0059360921538124485, "clip_ratio/low_min": 0.0059360921538124485, "clip_ratio/region_mean": 0.007995516527444124, "completions/clipped_ratio": 0.2222222238779068, "completions/max_length": 3000.0, "completions/max_terminated_length": 2981.0, "completions/mean_length": 2085.638916015625, "completions/mean_terminated_length": 1824.3929443359375, "completions/min_length": 956.0, "completions/min_terminated_length": 956.0, "entropy": 0.18983352681001028, "epoch": 0.005555555555555556, "frac_reward_zero_std": 0.0, "grad_norm": 0.2806412875652313, "learning_rate": 4.615384615384616e-05, "loss": 0.0294, "num_tokens": 1128414.0, "reward": -0.6674352288246155, "reward_std": 0.8722909092903137, "rewards/hint_following/mean": 0.02777777798473835, "rewards/hint_following/std": 0.6963624954223633, "rewards/length_penalty/mean": -0.6952130198478699, "rewards/length_penalty/std": 0.23719921708106995, "sampling/importance_sampling_ratio/max": 2.7140581607818604, "sampling/importance_sampling_ratio/mean": 0.9935481548309326, "sampling/importance_sampling_ratio/min": 0.19567115604877472, "sampling/sampling_logp_difference/max": 1.6313197612762451, "sampling/sampling_logp_difference/mean": 0.02119005285203457, "step": 13, "step_time": 61.55426321423147 }, { "clip_ratio/high_max": 0.0014092368267786999, "clip_ratio/high_mean": 0.0014092368267786999, "clip_ratio/low_mean": 0.007215868448838592, "clip_ratio/low_min": 0.007215868448838592, "clip_ratio/region_mean": 0.008625105411435166, "completions/clipped_ratio": 0.1944444477558136, "completions/max_length": 3000.0, "completions/max_terminated_length": 2941.0, "completions/mean_length": 1680.3055419921875, "completions/mean_terminated_length": 1361.7586669921875, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "entropy": 0.20419775694608688, "epoch": 0.005982905982905983, "frac_reward_zero_std": 0.0, "grad_norm": 0.08586689084768295, "learning_rate": 5e-05, "loss": 0.0681, "num_tokens": 1193759.0, "reward": -0.5878795981407166, "reward_std": 0.8454972505569458, "rewards/hint_following/mean": -0.02777777798473835, "rewards/hint_following/std": 0.6087979674339294, "rewards/length_penalty/mean": -0.5601018071174622, "rewards/length_penalty/std": 0.31150683760643005, "sampling/importance_sampling_ratio/max": 2.1082680225372314, "sampling/importance_sampling_ratio/mean": 0.992834746837616, "sampling/importance_sampling_ratio/min": 0.3020838499069214, "sampling/sampling_logp_difference/max": 1.1970505714416504, "sampling/sampling_logp_difference/mean": 0.02261131815612316, "step": 14, "step_time": 53.72449496493209 }, { "clip_ratio/high_max": 0.007441268069669604, "clip_ratio/high_mean": 0.007441268069669604, "clip_ratio/low_mean": 0.002903422418360909, "clip_ratio/low_min": 0.002903422418360909, "clip_ratio/region_mean": 0.01034469079847137, "completions/clipped_ratio": 0.25, "completions/max_length": 3000.0, "completions/max_terminated_length": 2906.0, "completions/mean_length": 2191.111083984375, "completions/mean_terminated_length": 1921.4814453125, "completions/min_length": 397.0, "completions/min_terminated_length": 397.0, "entropy": 0.22065167874097824, "epoch": 0.00641025641025641, "frac_reward_zero_std": 0.0, "grad_norm": 0.07726756483316422, "learning_rate": 5e-05, "loss": 0.0177, "num_tokens": 1279125.0, "reward": -0.7859259247779846, "reward_std": 0.7896111607551575, "rewards/hint_following/mean": -0.0555555559694767, "rewards/hint_following/std": 0.6737716794013977, "rewards/length_penalty/mean": -0.7303703427314758, "rewards/length_penalty/std": 0.2772618532180786, "sampling/importance_sampling_ratio/max": 2.1340627670288086, "sampling/importance_sampling_ratio/mean": 0.9917824268341064, "sampling/importance_sampling_ratio/min": 0.07570598274469376, "sampling/sampling_logp_difference/max": 2.5808980464935303, "sampling/sampling_logp_difference/mean": 0.023916445672512054, "step": 15, "step_time": 59.183635055902414 }, { "clip_ratio/high_max": 0.007959824210653702, "clip_ratio/high_mean": 0.007959824210653702, "clip_ratio/low_mean": 0.0026605394668877125, "clip_ratio/low_min": 0.0026605394668877125, "clip_ratio/region_mean": 0.010620363832761845, "completions/clipped_ratio": 0.1388888955116272, "completions/max_length": 3000.0, "completions/max_terminated_length": 2836.0, "completions/mean_length": 1801.9722900390625, "completions/mean_terminated_length": 1612.0, "completions/min_length": 660.0, "completions/min_terminated_length": 660.0, "entropy": 0.2382892519235611, "epoch": 0.006837606837606838, "frac_reward_zero_std": 0.0, "grad_norm": 0.07441020756959915, "learning_rate": 5e-05, "loss": 0.1035, "num_tokens": 1351580.0, "reward": -0.2951018214225769, "reward_std": 0.9211878180503845, "rewards/hint_following/mean": 0.3055555522441864, "rewards/hint_following/std": 0.7099072337150574, "rewards/length_penalty/mean": -0.6006573438644409, "rewards/length_penalty/std": 0.2538762390613556, "sampling/importance_sampling_ratio/max": 2.141491413116455, "sampling/importance_sampling_ratio/mean": 0.9912582039833069, "sampling/importance_sampling_ratio/min": 0.3961084187030792, "sampling/sampling_logp_difference/max": 0.9260673522949219, "sampling/sampling_logp_difference/mean": 0.024969136342406273, "step": 16, "step_time": 58.414970112848096 }, { "clip_ratio/high_max": 0.00782025900358955, "clip_ratio/high_mean": 0.00782025900358955, "clip_ratio/low_mean": 0.0016357484661663573, "clip_ratio/low_min": 0.0016357484661663573, "clip_ratio/region_mean": 0.00945600758617123, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 3000.0, "completions/max_terminated_length": 2959.0, "completions/mean_length": 1858.02783203125, "completions/mean_terminated_length": 1754.212158203125, "completions/min_length": 523.0, "completions/min_terminated_length": 523.0, "entropy": 0.2335476577281952, "epoch": 0.007264957264957265, "frac_reward_zero_std": 0.0, "grad_norm": 0.06890398263931274, "learning_rate": 5e-05, "loss": 0.0923, "num_tokens": 1425669.0, "reward": 0.10287962108850479, "reward_std": 0.7695233225822449, "rewards/hint_following/mean": 0.7222222089767456, "rewards/hint_following/std": 0.6146363019943237, "rewards/length_penalty/mean": -0.619342565536499, "rewards/length_penalty/std": 0.2540966868400574, "sampling/importance_sampling_ratio/max": 2.8489930629730225, "sampling/importance_sampling_ratio/mean": 0.9912996888160706, "sampling/importance_sampling_ratio/min": 0.3177354037761688, "sampling/sampling_logp_difference/max": 1.1465363502502441, "sampling/sampling_logp_difference/mean": 0.0239370446652174, "step": 17, "step_time": 56.41279660095461 }, { "clip_ratio/high_max": 0.008962325053289533, "clip_ratio/high_mean": 0.008962325053289533, "clip_ratio/low_mean": 0.001638414493451516, "clip_ratio/low_min": 0.001638414493451516, "clip_ratio/region_mean": 0.010600739469130835, "completions/clipped_ratio": 0.2222222238779068, "completions/max_length": 3000.0, "completions/max_terminated_length": 2681.0, "completions/mean_length": 1895.4722900390625, "completions/mean_terminated_length": 1579.8929443359375, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "entropy": 0.2237540160616239, "epoch": 0.007692307692307693, "frac_reward_zero_std": 0.0, "grad_norm": 0.07782972604036331, "learning_rate": 5e-05, "loss": 0.0547, "num_tokens": 1501340.0, "reward": -0.3818241059780121, "reward_std": 0.9488492608070374, "rewards/hint_following/mean": 0.25, "rewards/hint_following/std": 0.8062257766723633, "rewards/length_penalty/mean": -0.6318240761756897, "rewards/length_penalty/std": 0.2734360098838806, "sampling/importance_sampling_ratio/max": 2.2928225994110107, "sampling/importance_sampling_ratio/mean": 0.9916783571243286, "sampling/importance_sampling_ratio/min": 0.3001212775707245, "sampling/sampling_logp_difference/max": 1.203568696975708, "sampling/sampling_logp_difference/mean": 0.023737527430057526, "step": 18, "step_time": 58.57279848307371 }, { "clip_ratio/high_max": 0.0038812818626562753, "clip_ratio/high_mean": 0.0038812818626562753, "clip_ratio/low_mean": 0.005094039059864978, "clip_ratio/low_min": 0.005094039059864978, "clip_ratio/region_mean": 0.00897532096132636, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 3000.0, "completions/max_terminated_length": 2937.0, "completions/mean_length": 2020.138916015625, "completions/mean_terminated_length": 1824.166748046875, "completions/min_length": 765.0, "completions/min_terminated_length": 765.0, "entropy": 0.2477496862411499, "epoch": 0.00811965811965812, "frac_reward_zero_std": 0.0, "grad_norm": 0.07387188822031021, "learning_rate": 5e-05, "loss": 0.0524, "num_tokens": 1581145.0, "reward": -0.39560189843177795, "reward_std": 0.9154050350189209, "rewards/hint_following/mean": 0.2777777910232544, "rewards/hint_following/std": 0.7410845756530762, "rewards/length_penalty/mean": -0.67337965965271, "rewards/length_penalty/std": 0.24809792637825012, "sampling/importance_sampling_ratio/max": 2.1414597034454346, "sampling/importance_sampling_ratio/mean": 0.9909990429878235, "sampling/importance_sampling_ratio/min": 0.250805139541626, "sampling/sampling_logp_difference/max": 1.383078932762146, "sampling/sampling_logp_difference/mean": 0.025834236294031143, "step": 19, "step_time": 58.03654656501021 }, { "clip_ratio/high_max": 0.0034701917708540955, "clip_ratio/high_mean": 0.0034701917708540955, "clip_ratio/low_mean": 0.004899940686300397, "clip_ratio/low_min": 0.004899940686300397, "clip_ratio/region_mean": 0.00837013234073917, "completions/clipped_ratio": 0.2777777910232544, "completions/max_length": 3000.0, "completions/max_terminated_length": 2645.0, "completions/mean_length": 1926.1666259765625, "completions/mean_terminated_length": 1513.1539306640625, "completions/min_length": 1111.0, "completions/min_terminated_length": 1111.0, "entropy": 0.2643888195355733, "epoch": 0.008547008547008548, "frac_reward_zero_std": 0.0, "grad_norm": 0.07248850166797638, "learning_rate": 5e-05, "loss": 0.0748, "num_tokens": 1658131.0, "reward": -0.25316667556762695, "reward_std": 1.1320863962173462, "rewards/hint_following/mean": 0.3888888955116272, "rewards/hint_following/std": 0.903256893157959, "rewards/length_penalty/mean": -0.6420555114746094, "rewards/length_penalty/std": 0.2418719381093979, "sampling/importance_sampling_ratio/max": 2.639763593673706, "sampling/importance_sampling_ratio/mean": 0.990202009677887, "sampling/importance_sampling_ratio/min": 0.3388563394546509, "sampling/sampling_logp_difference/max": 1.082179069519043, "sampling/sampling_logp_difference/mean": 0.026784788817167282, "step": 20, "step_time": 59.42642685002647 }, { "clip_ratio/high_max": 0.0018190126866102219, "clip_ratio/high_mean": 0.0018190126866102219, "clip_ratio/low_mean": 0.005009224289096892, "clip_ratio/low_min": 0.005009224289096892, "clip_ratio/region_mean": 0.006828237092122436, "completions/clipped_ratio": 0.3611111044883728, "completions/max_length": 3000.0, "completions/max_terminated_length": 2893.0, "completions/mean_length": 2087.0556640625, "completions/mean_terminated_length": 1571.04345703125, "completions/min_length": 621.0, "completions/min_terminated_length": 621.0, "entropy": 0.23874077697594961, "epoch": 0.008974358974358974, "frac_reward_zero_std": 0.0, "grad_norm": 0.07083278149366379, "learning_rate": 5e-05, "loss": 0.0432, "num_tokens": 1741971.0, "reward": -0.7512407302856445, "reward_std": 1.0566543340682983, "rewards/hint_following/mean": -0.0555555559694767, "rewards/hint_following/std": 0.826159656047821, "rewards/length_penalty/mean": -0.6956851482391357, "rewards/length_penalty/std": 0.29789718985557556, "sampling/importance_sampling_ratio/max": 2.2389557361602783, "sampling/importance_sampling_ratio/mean": 0.9909273386001587, "sampling/importance_sampling_ratio/min": 0.1903061866760254, "sampling/sampling_logp_difference/max": 1.659121036529541, "sampling/sampling_logp_difference/mean": 0.024532614275813103, "step": 21, "step_time": 62.53779820702039 }, { "clip_ratio/high_max": 0.002690806732668231, "clip_ratio/high_mean": 0.002690806732668231, "clip_ratio/low_mean": 0.0009356096270494163, "clip_ratio/low_min": 0.0009356096270494163, "clip_ratio/region_mean": 0.003626416379120201, "completions/clipped_ratio": 0.1111111119389534, "completions/max_length": 3000.0, "completions/max_terminated_length": 2897.0, "completions/mean_length": 1785.02783203125, "completions/mean_terminated_length": 1633.15625, "completions/min_length": 687.0, "completions/min_terminated_length": 687.0, "entropy": 0.2271458531419436, "epoch": 0.009401709401709401, "frac_reward_zero_std": 0.0, "grad_norm": 0.06376982480287552, "learning_rate": 5e-05, "loss": 0.0558, "num_tokens": 1812208.0, "reward": -0.12278702855110168, "reward_std": 0.8545697927474976, "rewards/hint_following/mean": 0.4722222089767456, "rewards/hint_following/std": 0.6963624358177185, "rewards/length_penalty/mean": -0.5950092673301697, "rewards/length_penalty/std": 0.2556186616420746, "sampling/importance_sampling_ratio/max": 2.0840954780578613, "sampling/importance_sampling_ratio/mean": 0.9917669892311096, "sampling/importance_sampling_ratio/min": 0.4361201226711273, "sampling/sampling_logp_difference/max": 0.8298375606536865, "sampling/sampling_logp_difference/mean": 0.02324368618428707, "step": 22, "step_time": 54.28889209416229 }, { "clip_ratio/high_max": 0.00034443634406973916, "clip_ratio/high_mean": 0.00034443634406973916, "clip_ratio/low_mean": 0.001613353689511617, "clip_ratio/low_min": 0.001613353689511617, "clip_ratio/region_mean": 0.001957789994776249, "completions/clipped_ratio": 0.0, "completions/max_length": 2755.0, "completions/max_terminated_length": 2755.0, "completions/mean_length": 1515.5555419921875, "completions/mean_terminated_length": 1515.5555419921875, "completions/min_length": 761.0, "completions/min_terminated_length": 761.0, "entropy": 0.26160697638988495, "epoch": 0.009829059829059829, "frac_reward_zero_std": 0.0, "grad_norm": 0.05521637201309204, "learning_rate": 5e-05, "loss": 0.0467, "num_tokens": 1873452.0, "reward": 0.18925924599170685, "reward_std": 0.6213830709457397, "rewards/hint_following/mean": 0.6944444179534912, "rewards/hint_following/std": 0.467176616191864, "rewards/length_penalty/mean": -0.5051851868629456, "rewards/length_penalty/std": 0.1804182529449463, "sampling/importance_sampling_ratio/max": 2.6960017681121826, "sampling/importance_sampling_ratio/mean": 0.9903501272201538, "sampling/importance_sampling_ratio/min": 0.2622704803943634, "sampling/sampling_logp_difference/max": 1.33837890625, "sampling/sampling_logp_difference/mean": 0.026966720819473267, "step": 23, "step_time": 49.74234241596423 }, { "clip_ratio/high_max": 0.009169099697222313, "clip_ratio/high_mean": 0.009169099697222313, "clip_ratio/low_mean": 0.0011019935482181609, "clip_ratio/low_min": 0.0011019935482181609, "clip_ratio/region_mean": 0.010271093342453241, "completions/clipped_ratio": 0.02777777798473835, "completions/max_length": 3000.0, "completions/max_terminated_length": 2438.0, "completions/mean_length": 1354.77783203125, "completions/mean_terminated_length": 1307.771484375, "completions/min_length": 358.0, "completions/min_terminated_length": 358.0, "entropy": 0.28231214980284375, "epoch": 0.010256410256410256, "frac_reward_zero_std": 0.0, "grad_norm": 0.061877232044935226, "learning_rate": 5e-05, "loss": 0.0513, "num_tokens": 1928578.0, "reward": 0.0761851817369461, "reward_std": 0.6364034414291382, "rewards/hint_following/mean": 0.5277777910232544, "rewards/hint_following/std": 0.559903621673584, "rewards/length_penalty/mean": -0.4515925645828247, "rewards/length_penalty/std": 0.2236867994070053, "sampling/importance_sampling_ratio/max": 2.3720901012420654, "sampling/importance_sampling_ratio/mean": 0.9896314144134521, "sampling/importance_sampling_ratio/min": 0.3866640329360962, "sampling/sampling_logp_difference/max": 0.9501991271972656, "sampling/sampling_logp_difference/mean": 0.028320252895355225, "step": 24, "step_time": 51.51962701487355 }, { "clip_ratio/high_max": 0.007340530709673961, "clip_ratio/high_mean": 0.007340530709673961, "clip_ratio/low_mean": 0.002632848802022636, "clip_ratio/low_min": 0.002632848802022636, "clip_ratio/region_mean": 0.00997337931767106, "completions/clipped_ratio": 0.0555555559694767, "completions/max_length": 3000.0, "completions/max_terminated_length": 2833.0, "completions/mean_length": 1369.52783203125, "completions/mean_terminated_length": 1273.61767578125, "completions/min_length": 489.0, "completions/min_terminated_length": 489.0, "entropy": 0.24688356121381125, "epoch": 0.010683760683760684, "frac_reward_zero_std": 0.0, "grad_norm": 0.0653451606631279, "learning_rate": 5e-05, "loss": 0.0562, "num_tokens": 1983563.0, "reward": -0.15095369517803192, "reward_std": 0.692315936088562, "rewards/hint_following/mean": 0.3055555522441864, "rewards/hint_following/std": 0.576662540435791, "rewards/length_penalty/mean": -0.4565092623233795, "rewards/length_penalty/std": 0.22569186985492706, "sampling/importance_sampling_ratio/max": 2.21685791015625, "sampling/importance_sampling_ratio/mean": 0.9907597899436951, "sampling/importance_sampling_ratio/min": 0.26015329360961914, "sampling/sampling_logp_difference/max": 1.3464841842651367, "sampling/sampling_logp_difference/mean": 0.02600344829261303, "step": 25, "step_time": 53.30105395393912 }, { "clip_ratio/high_max": 0.005985865951515734, "clip_ratio/high_mean": 0.005985865951515734, "clip_ratio/low_mean": 0.0026564535413247845, "clip_ratio/low_min": 0.0026564535413247845, "clip_ratio/region_mean": 0.008642319512243072, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 3000.0, "completions/max_terminated_length": 2935.0, "completions/mean_length": 1874.6944580078125, "completions/mean_terminated_length": 1772.39404296875, "completions/min_length": 921.0, "completions/min_terminated_length": 921.0, "entropy": 0.24719314028819403, "epoch": 0.011111111111111112, "frac_reward_zero_std": 0.0, "grad_norm": 0.07119674980640411, "learning_rate": 5e-05, "loss": 0.0874, "num_tokens": 2058072.0, "reward": -0.06934259831905365, "reward_std": 0.8091042637825012, "rewards/hint_following/mean": 0.5555555820465088, "rewards/hint_following/std": 0.6522245407104492, "rewards/length_penalty/mean": -0.6248981952667236, "rewards/length_penalty/std": 0.21120469272136688, "sampling/importance_sampling_ratio/max": 2.3882269859313965, "sampling/importance_sampling_ratio/mean": 0.9909408688545227, "sampling/importance_sampling_ratio/min": 0.3095887005329132, "sampling/sampling_logp_difference/max": 1.1725106239318848, "sampling/sampling_logp_difference/mean": 0.024691704660654068, "step": 26, "step_time": 55.06593968113884 }, { "clip_ratio/high_max": 0.006906990543939173, "clip_ratio/high_mean": 0.006906990543939173, "clip_ratio/low_mean": 0.002326829261922588, "clip_ratio/low_min": 0.002326829261922588, "clip_ratio/region_mean": 0.009233820019289851, "completions/clipped_ratio": 0.1388888955116272, "completions/max_length": 3000.0, "completions/max_terminated_length": 2980.0, "completions/mean_length": 1767.5833740234375, "completions/mean_terminated_length": 1568.806396484375, "completions/min_length": 614.0, "completions/min_terminated_length": 614.0, "entropy": 0.25413808474938077, "epoch": 0.011538461538461539, "frac_reward_zero_std": 0.0, "grad_norm": 0.06020393222570419, "learning_rate": 5e-05, "loss": 0.0678, "num_tokens": 2126523.0, "reward": -0.3114166855812073, "reward_std": 0.8995095491409302, "rewards/hint_following/mean": 0.2777777910232544, "rewards/hint_following/std": 0.7014724016189575, "rewards/length_penalty/mean": -0.5891944169998169, "rewards/length_penalty/std": 0.24767503142356873, "sampling/importance_sampling_ratio/max": 2.9181816577911377, "sampling/importance_sampling_ratio/mean": 0.9907821416854858, "sampling/importance_sampling_ratio/min": 0.28627336025238037, "sampling/sampling_logp_difference/max": 1.2508081197738647, "sampling/sampling_logp_difference/mean": 0.025369901210069656, "step": 27, "step_time": 54.32182147912681 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3333333432674408, "completions/max_length": 3000.0, "completions/max_terminated_length": 1529.0, "completions/mean_length": 1610.5833740234375, "completions/mean_terminated_length": 915.875, "completions/min_length": 478.0, "completions/min_terminated_length": 478.0, "entropy": 0.19960021724303564, "epoch": 0.011965811965811967, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 2192742.0, "reward": -0.8701944351196289, "reward_std": 0.8150903582572937, "rewards/hint_following/mean": -0.3333333432674408, "rewards/hint_following/std": 0.47809144854545593, "rewards/length_penalty/mean": -0.5368611216545105, "rewards/length_penalty/std": 0.3438311517238617, "sampling/importance_sampling_ratio/max": 2.0817480087280273, "sampling/importance_sampling_ratio/mean": 0.9929863214492798, "sampling/importance_sampling_ratio/min": 0.09024836868047714, "sampling/sampling_logp_difference/max": 2.4051897525787354, "sampling/sampling_logp_difference/mean": 0.02168331854045391, "step": 28, "step_time": 56.78273802890908 }, { "clip_ratio/high_max": 0.001870962364288668, "clip_ratio/high_mean": 0.001870962364288668, "clip_ratio/low_mean": 0.004373128215471904, "clip_ratio/low_min": 0.004373128215471904, "clip_ratio/region_mean": 0.006244090540955464, "completions/clipped_ratio": 0.1944444477558136, "completions/max_length": 3000.0, "completions/max_terminated_length": 2978.0, "completions/mean_length": 1729.77783203125, "completions/mean_terminated_length": 1423.17236328125, "completions/min_length": 529.0, "completions/min_terminated_length": 529.0, "entropy": 0.28223206599553424, "epoch": 0.012393162393162393, "frac_reward_zero_std": 0.0, "grad_norm": 0.07922516763210297, "learning_rate": 5e-05, "loss": 0.0378, "num_tokens": 2262334.0, "reward": -0.3265925943851471, "reward_std": 0.9465053081512451, "rewards/hint_following/mean": 0.25, "rewards/hint_following/std": 0.7699722051620483, "rewards/length_penalty/mean": -0.5765926241874695, "rewards/length_penalty/std": 0.28062164783477783, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9894134402275085, "sampling/importance_sampling_ratio/min": 0.29406988620758057, "sampling/sampling_logp_difference/max": 1.760861873626709, "sampling/sampling_logp_difference/mean": 0.026971129700541496, "step": 29, "step_time": 55.79818298702594 }, { "clip_ratio/high_max": 0.001682481961324811, "clip_ratio/high_mean": 0.001682481961324811, "clip_ratio/low_mean": 0.00047758713481016457, "clip_ratio/low_min": 0.00047758713481016457, "clip_ratio/region_mean": 0.002160069086433699, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 3000.0, "completions/max_terminated_length": 2747.0, "completions/mean_length": 1684.638916015625, "completions/mean_terminated_length": 1421.5667724609375, "completions/min_length": 805.0, "completions/min_terminated_length": 805.0, "entropy": 0.2508200804392497, "epoch": 0.01282051282051282, "frac_reward_zero_std": 0.0, "grad_norm": 0.06276588886976242, "learning_rate": 5e-05, "loss": 0.0312, "num_tokens": 2331153.0, "reward": 0.02178703434765339, "reward_std": 0.9737120270729065, "rewards/hint_following/mean": 0.5833333134651184, "rewards/hint_following/std": 0.7699722051620483, "rewards/length_penalty/mean": -0.5615463256835938, "rewards/length_penalty/std": 0.233141228556633, "sampling/importance_sampling_ratio/max": 2.9634652137756348, "sampling/importance_sampling_ratio/mean": 0.9905918836593628, "sampling/importance_sampling_ratio/min": 0.2952629625797272, "sampling/sampling_logp_difference/max": 1.2198889255523682, "sampling/sampling_logp_difference/mean": 0.024356598034501076, "step": 30, "step_time": 56.55290222284384 }, { "clip_ratio/high_max": 0.008109931523601214, "clip_ratio/high_mean": 0.008109931523601214, "clip_ratio/low_mean": 0.0004286583377203594, "clip_ratio/low_min": 0.0004286583377203594, "clip_ratio/region_mean": 0.008538589735204974, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 3000.0, "completions/max_terminated_length": 2567.0, "completions/mean_length": 1786.2222900390625, "completions/mean_terminated_length": 1675.8787841796875, "completions/min_length": 468.0, "completions/min_terminated_length": 468.0, "entropy": 0.2908742278814316, "epoch": 0.013247863247863248, "frac_reward_zero_std": 0.0, "grad_norm": 0.07696390151977539, "learning_rate": 5e-05, "loss": 0.0561, "num_tokens": 2403953.0, "reward": -0.0954074040055275, "reward_std": 0.6877580285072327, "rewards/hint_following/mean": 0.5, "rewards/hint_following/std": 0.6546536684036255, "rewards/length_penalty/mean": -0.5954073667526245, "rewards/length_penalty/std": 0.22547654807567596, "sampling/importance_sampling_ratio/max": 2.491806745529175, "sampling/importance_sampling_ratio/mean": 0.9886121153831482, "sampling/importance_sampling_ratio/min": 0.16667605936527252, "sampling/sampling_logp_difference/max": 1.7917031049728394, "sampling/sampling_logp_difference/mean": 0.02771041914820671, "step": 31, "step_time": 58.1784061481012 }, { "clip_ratio/high_max": 0.0027225150261074305, "clip_ratio/high_mean": 0.0027225150261074305, "clip_ratio/low_mean": 0.003243642975576222, "clip_ratio/low_min": 0.003243642975576222, "clip_ratio/region_mean": 0.00596615804048876, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 3000.0, "completions/max_terminated_length": 2917.0, "completions/mean_length": 2085.916748046875, "completions/mean_terminated_length": 1903.10009765625, "completions/min_length": 1014.0, "completions/min_terminated_length": 1014.0, "entropy": 0.278283953666687, "epoch": 0.013675213675213675, "frac_reward_zero_std": 0.0, "grad_norm": 0.06229299306869507, "learning_rate": 5e-05, "loss": 0.0363, "num_tokens": 2486342.0, "reward": -0.36197224259376526, "reward_std": 0.8716595768928528, "rewards/hint_following/mean": 0.3333333432674408, "rewards/hint_following/std": 0.7559289932250977, "rewards/length_penalty/mean": -0.695305585861206, "rewards/length_penalty/std": 0.1993320733308792, "sampling/importance_sampling_ratio/max": 2.49179744720459, "sampling/importance_sampling_ratio/mean": 0.9892889261245728, "sampling/importance_sampling_ratio/min": 0.19078373908996582, "sampling/sampling_logp_difference/max": 1.6566147804260254, "sampling/sampling_logp_difference/mean": 0.02677111327648163, "step": 32, "step_time": 59.783586613833904 }, { "clip_ratio/high_max": 0.0016072407306637615, "clip_ratio/high_mean": 0.0016072407306637615, "clip_ratio/low_mean": 0.0016927721832568448, "clip_ratio/low_min": 0.0016927721832568448, "clip_ratio/region_mean": 0.003300012855712945, "completions/clipped_ratio": 0.0, "completions/max_length": 2629.0, "completions/max_terminated_length": 2629.0, "completions/mean_length": 1464.5833740234375, "completions/mean_terminated_length": 1464.5833740234375, "completions/min_length": 577.0, "completions/min_terminated_length": 577.0, "entropy": 0.25622066855430603, "epoch": 0.014102564102564103, "frac_reward_zero_std": 0.0, "grad_norm": 0.05397763103246689, "learning_rate": 5e-05, "loss": 0.0059, "num_tokens": 2544023.0, "reward": -0.29375001788139343, "reward_std": 0.45689767599105835, "rewards/hint_following/mean": 0.1944444477558136, "rewards/hint_following/std": 0.4013865292072296, "rewards/length_penalty/mean": -0.48819446563720703, "rewards/length_penalty/std": 0.14601026475429535, "sampling/importance_sampling_ratio/max": 2.081763982772827, "sampling/importance_sampling_ratio/mean": 0.9904037117958069, "sampling/importance_sampling_ratio/min": 0.339209645986557, "sampling/sampling_logp_difference/max": 1.08113694190979, "sampling/sampling_logp_difference/mean": 0.02619028650224209, "step": 33, "step_time": 49.13755757326726 }, { "clip_ratio/high_max": 0.001959889098846664, "clip_ratio/high_mean": 0.001959889098846664, "clip_ratio/low_mean": 0.0017016992593804996, "clip_ratio/low_min": 0.0017016992593804996, "clip_ratio/region_mean": 0.0036615883776297173, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 3000.0, "completions/max_terminated_length": 2814.0, "completions/mean_length": 1581.1666259765625, "completions/mean_terminated_length": 1297.4000244140625, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "entropy": 0.25040922313928604, "epoch": 0.01452991452991453, "frac_reward_zero_std": 0.0, "grad_norm": 0.06385543942451477, "learning_rate": 5e-05, "loss": 0.046, "num_tokens": 2606177.0, "reward": -0.0826110988855362, "reward_std": 0.9639437198638916, "rewards/hint_following/mean": 0.4444444477558136, "rewards/hint_following/std": 0.7725447416305542, "rewards/length_penalty/mean": -0.5270555019378662, "rewards/length_penalty/std": 0.2994069457054138, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9912077784538269, "sampling/importance_sampling_ratio/min": 0.31699931621551514, "sampling/sampling_logp_difference/max": 1.2634482383728027, "sampling/sampling_logp_difference/mean": 0.02445373497903347, "step": 34, "step_time": 54.58746225386858 }, { "clip_ratio/high_max": 0.0011685320253794391, "clip_ratio/high_mean": 0.0011685320253794391, "clip_ratio/low_mean": 0.0014686054394890864, "clip_ratio/low_min": 0.0014686054394890864, "clip_ratio/region_mean": 0.0026371375230761864, "completions/clipped_ratio": 0.0, "completions/max_length": 2743.0, "completions/max_terminated_length": 2743.0, "completions/mean_length": 1370.4722900390625, "completions/mean_terminated_length": 1370.4722900390625, "completions/min_length": 650.0, "completions/min_terminated_length": 650.0, "entropy": 0.26418235152959824, "epoch": 0.014957264957264958, "frac_reward_zero_std": 0.0, "grad_norm": 0.057452693581581116, "learning_rate": 5e-05, "loss": 0.01, "num_tokens": 2661580.0, "reward": 0.15428705513477325, "reward_std": 0.5879056453704834, "rewards/hint_following/mean": 0.6111111044883728, "rewards/hint_following/std": 0.49441322684288025, "rewards/length_penalty/mean": -0.45682409405708313, "rewards/length_penalty/std": 0.1867014318704605, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9900287985801697, "sampling/importance_sampling_ratio/min": 0.15399949252605438, "sampling/sampling_logp_difference/max": 1.8708059787750244, "sampling/sampling_logp_difference/mean": 0.027402520179748535, "step": 35, "step_time": 47.69401362503413 }, { "clip_ratio/high_max": 0.0057743463742857175, "clip_ratio/high_mean": 0.0057743463742857175, "clip_ratio/low_mean": 0.0029491251916624606, "clip_ratio/low_min": 0.0029491251916624606, "clip_ratio/region_mean": 0.00872347146893541, "completions/clipped_ratio": 0.1111111119389534, "completions/max_length": 3000.0, "completions/max_terminated_length": 2813.0, "completions/mean_length": 1503.6944580078125, "completions/mean_terminated_length": 1316.65625, "completions/min_length": 456.0, "completions/min_terminated_length": 456.0, "entropy": 0.2601601282755534, "epoch": 0.015384615384615385, "frac_reward_zero_std": 0.0, "grad_norm": 0.06052026152610779, "learning_rate": 5e-05, "loss": 0.0217, "num_tokens": 2721131.0, "reward": -0.44567596912384033, "reward_std": 0.6641168594360352, "rewards/hint_following/mean": 0.0555555559694767, "rewards/hint_following/std": 0.5315446853637695, "rewards/length_penalty/mean": -0.5012314915657043, "rewards/length_penalty/std": 0.2637265622615814, "sampling/importance_sampling_ratio/max": 2.678779125213623, "sampling/importance_sampling_ratio/mean": 0.98993980884552, "sampling/importance_sampling_ratio/min": 0.3602158725261688, "sampling/sampling_logp_difference/max": 1.0210518836975098, "sampling/sampling_logp_difference/mean": 0.02683359384536743, "step": 36, "step_time": 52.9094661710551 }, { "clip_ratio/high_max": 0.002038249202693502, "clip_ratio/high_mean": 0.002038249202693502, "clip_ratio/low_mean": 0.00521218675809602, "clip_ratio/low_min": 0.00521218675809602, "clip_ratio/region_mean": 0.007250435883179307, "completions/clipped_ratio": 0.0, "completions/max_length": 2391.0, "completions/max_terminated_length": 2391.0, "completions/mean_length": 1279.638916015625, "completions/mean_terminated_length": 1279.638916015625, "completions/min_length": 508.0, "completions/min_terminated_length": 508.0, "entropy": 0.26387400925159454, "epoch": 0.01581196581196581, "frac_reward_zero_std": 0.0, "grad_norm": 0.0660337284207344, "learning_rate": 5e-05, "loss": 0.0072, "num_tokens": 2774056.0, "reward": 0.12900924682617188, "reward_std": 0.5431578159332275, "rewards/hint_following/mean": 0.5555555820465088, "rewards/hint_following/std": 0.5039526224136353, "rewards/length_penalty/mean": -0.4265463054180145, "rewards/length_penalty/std": 0.12839393317699432, "sampling/importance_sampling_ratio/max": 2.7058398723602295, "sampling/importance_sampling_ratio/mean": 0.9903796911239624, "sampling/importance_sampling_ratio/min": 0.1684948354959488, "sampling/sampling_logp_difference/max": 1.7808502912521362, "sampling/sampling_logp_difference/mean": 0.027670899406075478, "step": 37, "step_time": 42.42564612603746 }, { "clip_ratio/high_max": 0.005249147206389655, "clip_ratio/high_mean": 0.005249147206389655, "clip_ratio/low_mean": 0.0019288610492367297, "clip_ratio/low_min": 0.0019288610492367297, "clip_ratio/region_mean": 0.007178008245925109, "completions/clipped_ratio": 0.1388888955116272, "completions/max_length": 3000.0, "completions/max_terminated_length": 2773.0, "completions/mean_length": 1562.9166259765625, "completions/mean_terminated_length": 1331.1290283203125, "completions/min_length": 508.0, "completions/min_terminated_length": 508.0, "entropy": 0.2739727521936099, "epoch": 0.01623931623931624, "frac_reward_zero_std": 0.0, "grad_norm": 0.05931999906897545, "learning_rate": 5e-05, "loss": 0.0111, "num_tokens": 2836603.0, "reward": 0.034583330154418945, "reward_std": 0.9446364045143127, "rewards/hint_following/mean": 0.5555555820465088, "rewards/hint_following/std": 0.7346308827400208, "rewards/length_penalty/mean": -0.5209722518920898, "rewards/length_penalty/std": 0.2565736174583435, "sampling/importance_sampling_ratio/max": 2.9026551246643066, "sampling/importance_sampling_ratio/mean": 0.9896891117095947, "sampling/importance_sampling_ratio/min": 0.1531003713607788, "sampling/sampling_logp_difference/max": 1.8766615390777588, "sampling/sampling_logp_difference/mean": 0.02661670744419098, "step": 38, "step_time": 54.65668713499326 }, { "clip_ratio/high_max": 0.0073317300993949175, "clip_ratio/high_mean": 0.0073317300993949175, "clip_ratio/low_mean": 0.0012036147963954136, "clip_ratio/low_min": 0.0012036147963954136, "clip_ratio/region_mean": 0.008535344852134585, "completions/clipped_ratio": 0.0, "completions/max_length": 2373.0, "completions/max_terminated_length": 2373.0, "completions/mean_length": 1587.7222900390625, "completions/mean_terminated_length": 1587.7222900390625, "completions/min_length": 887.0, "completions/min_terminated_length": 887.0, "entropy": 0.2784932255744934, "epoch": 0.016666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.05303090810775757, "learning_rate": 5e-05, "loss": -0.0225, "num_tokens": 2900313.0, "reward": 0.22075927257537842, "reward_std": 0.3945108652114868, "rewards/hint_following/mean": 0.75, "rewards/hint_following/std": 0.43915504217147827, "rewards/length_penalty/mean": -0.5292407274246216, "rewards/length_penalty/std": 0.10378893464803696, "sampling/importance_sampling_ratio/max": 2.234417200088501, "sampling/importance_sampling_ratio/mean": 0.9896702170372009, "sampling/importance_sampling_ratio/min": 0.22993366420269012, "sampling/sampling_logp_difference/max": 1.4699643850326538, "sampling/sampling_logp_difference/mean": 0.027140147984027863, "step": 39, "step_time": 45.66687847697176 }, { "clip_ratio/high_max": 0.008284316320593158, "clip_ratio/high_mean": 0.008284316320593158, "clip_ratio/low_mean": 0.0006285767303779721, "clip_ratio/low_min": 0.0006285767303779721, "clip_ratio/region_mean": 0.008912893012166023, "completions/clipped_ratio": 0.0, "completions/max_length": 2490.0, "completions/max_terminated_length": 2490.0, "completions/mean_length": 1358.638916015625, "completions/mean_terminated_length": 1358.638916015625, "completions/min_length": 668.0, "completions/min_terminated_length": 668.0, "entropy": 0.2635483493407567, "epoch": 0.017094017094017096, "frac_reward_zero_std": 0.0, "grad_norm": 0.061241693794727325, "learning_rate": 5e-05, "loss": -0.0001, "num_tokens": 2955062.0, "reward": 0.2693426012992859, "reward_std": 0.4795752167701721, "rewards/hint_following/mean": 0.7222222089767456, "rewards/hint_following/std": 0.45425674319267273, "rewards/length_penalty/mean": -0.4528796374797821, "rewards/length_penalty/std": 0.17493696510791779, "sampling/importance_sampling_ratio/max": 2.239511728286743, "sampling/importance_sampling_ratio/mean": 0.9903267621994019, "sampling/importance_sampling_ratio/min": 0.11606565117835999, "sampling/sampling_logp_difference/max": 2.153599262237549, "sampling/sampling_logp_difference/mean": 0.02778884768486023, "step": 40, "step_time": 44.78665240900591 }, { "clip_ratio/high_max": 0.0024162703969826302, "clip_ratio/high_mean": 0.0024162703969826302, "clip_ratio/low_mean": 0.004380864285243054, "clip_ratio/low_min": 0.004380864285243054, "clip_ratio/region_mean": 0.006797134643420577, "completions/clipped_ratio": 0.02777777798473835, "completions/max_length": 3000.0, "completions/max_terminated_length": 2931.0, "completions/mean_length": 1570.75, "completions/mean_terminated_length": 1529.914306640625, "completions/min_length": 437.0, "completions/min_terminated_length": 437.0, "entropy": 0.2851356367270152, "epoch": 0.01752136752136752, "frac_reward_zero_std": 0.0, "grad_norm": 0.06564202159643173, "learning_rate": 5e-05, "loss": 0.0083, "num_tokens": 3020207.0, "reward": 0.1708611100912094, "reward_std": 0.6461936831474304, "rewards/hint_following/mean": 0.6944444179534912, "rewards/hint_following/std": 0.524782657623291, "rewards/length_penalty/mean": -0.5235832929611206, "rewards/length_penalty/std": 0.23531726002693176, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9896803498268127, "sampling/importance_sampling_ratio/min": 0.2426270991563797, "sampling/sampling_logp_difference/max": 1.4162296056747437, "sampling/sampling_logp_difference/mean": 0.02950497530400753, "step": 41, "step_time": 55.908626700984314 }, { "clip_ratio/high_max": 0.0035200511144163706, "clip_ratio/high_mean": 0.0035200511144163706, "clip_ratio/low_mean": 0.0037633493387450776, "clip_ratio/low_min": 0.0037633493387450776, "clip_ratio/region_mean": 0.007283400433758895, "completions/clipped_ratio": 0.1944444477558136, "completions/max_length": 3000.0, "completions/max_terminated_length": 2916.0, "completions/mean_length": 1708.861083984375, "completions/mean_terminated_length": 1397.2069091796875, "completions/min_length": 601.0, "completions/min_terminated_length": 601.0, "entropy": 0.297603373726209, "epoch": 0.017948717948717947, "frac_reward_zero_std": 0.0, "grad_norm": 0.06374821066856384, "learning_rate": 5e-05, "loss": 0.0258, "num_tokens": 3087486.0, "reward": -0.40295371413230896, "reward_std": 0.9737837910652161, "rewards/hint_following/mean": 0.1666666716337204, "rewards/hint_following/std": 0.7367883920669556, "rewards/length_penalty/mean": -0.5696203708648682, "rewards/length_penalty/std": 0.28024521470069885, "sampling/importance_sampling_ratio/max": 2.632629871368408, "sampling/importance_sampling_ratio/mean": 0.9891336560249329, "sampling/importance_sampling_ratio/min": 0.16689430177211761, "sampling/sampling_logp_difference/max": 1.79039466381073, "sampling/sampling_logp_difference/mean": 0.029687926173210144, "step": 42, "step_time": 55.97696550388355 }, { "clip_ratio/high_max": 0.005934966184819738, "clip_ratio/high_mean": 0.005934966184819738, "clip_ratio/low_mean": 0.0023504976027955613, "clip_ratio/low_min": 0.0023504976027955613, "clip_ratio/region_mean": 0.008285463865225514, "completions/clipped_ratio": 0.0, "completions/max_length": 2141.0, "completions/max_terminated_length": 2141.0, "completions/mean_length": 1201.638916015625, "completions/mean_terminated_length": 1201.638916015625, "completions/min_length": 517.0, "completions/min_terminated_length": 517.0, "entropy": 0.29462653398513794, "epoch": 0.018376068376068377, "frac_reward_zero_std": 0.0, "grad_norm": 0.05552098527550697, "learning_rate": 5e-05, "loss": 0.0036, "num_tokens": 3135263.0, "reward": 0.12723149359226227, "reward_std": 0.5270769596099854, "rewards/hint_following/mean": 0.5277777910232544, "rewards/hint_following/std": 0.506309449672699, "rewards/length_penalty/mean": -0.40054628252983093, "rewards/length_penalty/std": 0.14752288162708282, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9892792701721191, "sampling/importance_sampling_ratio/min": 0.3035065531730652, "sampling/sampling_logp_difference/max": 1.192352056503296, "sampling/sampling_logp_difference/mean": 0.030870094895362854, "step": 43, "step_time": 37.3642987072235 }, { "clip_ratio/high_max": 0.0028009160111347833, "clip_ratio/high_mean": 0.0028009160111347833, "clip_ratio/low_mean": 0.003960246841112773, "clip_ratio/low_min": 0.003960246841112773, "clip_ratio/region_mean": 0.0067611627746373415, "completions/clipped_ratio": 0.2777777910232544, "completions/max_length": 3000.0, "completions/max_terminated_length": 2767.0, "completions/mean_length": 1834.611083984375, "completions/mean_terminated_length": 1386.3846435546875, "completions/min_length": 439.0, "completions/min_terminated_length": 439.0, "entropy": 0.2577643742163976, "epoch": 0.018803418803418803, "frac_reward_zero_std": 0.0, "grad_norm": 0.06315090507268906, "learning_rate": 5e-05, "loss": 0.0313, "num_tokens": 3208011.0, "reward": -0.22264817357063293, "reward_std": 1.158618688583374, "rewards/hint_following/mean": 0.3888888955116272, "rewards/hint_following/std": 0.903256893157959, "rewards/length_penalty/mean": -0.6115370988845825, "rewards/length_penalty/std": 0.3052543103694916, "sampling/importance_sampling_ratio/max": 2.770780324935913, "sampling/importance_sampling_ratio/mean": 0.9908032417297363, "sampling/importance_sampling_ratio/min": 0.14089906215667725, "sampling/sampling_logp_difference/max": 1.9597115516662598, "sampling/sampling_logp_difference/mean": 0.026477616280317307, "step": 44, "step_time": 57.508703485131264 }, { "clip_ratio/high_max": 0.0022431810890945294, "clip_ratio/high_mean": 0.0022431810890945294, "clip_ratio/low_mean": 0.0008466135962711027, "clip_ratio/low_min": 0.0008466135962711027, "clip_ratio/region_mean": 0.003089794685365632, "completions/clipped_ratio": 0.0, "completions/max_length": 2567.0, "completions/max_terminated_length": 2567.0, "completions/mean_length": 1403.611083984375, "completions/mean_terminated_length": 1403.611083984375, "completions/min_length": 791.0, "completions/min_terminated_length": 791.0, "entropy": 0.2975164105494817, "epoch": 0.019230769230769232, "frac_reward_zero_std": 0.0, "grad_norm": 0.0667228028178215, "learning_rate": 5e-05, "loss": 0.0231, "num_tokens": 3265633.0, "reward": 0.39324072003364563, "reward_std": 0.3923882842063904, "rewards/hint_following/mean": 0.8611111044883728, "rewards/hint_following/std": 0.35073620080947876, "rewards/length_penalty/mean": -0.4678703546524048, "rewards/length_penalty/std": 0.15880942344665527, "sampling/importance_sampling_ratio/max": 2.800001382827759, "sampling/importance_sampling_ratio/mean": 0.9887976050376892, "sampling/importance_sampling_ratio/min": 0.2656780183315277, "sampling/sampling_logp_difference/max": 1.325470209121704, "sampling/sampling_logp_difference/mean": 0.030811548233032227, "step": 45, "step_time": 45.27160808304325 }, { "clip_ratio/high_max": 0.007887479305888215, "clip_ratio/high_mean": 0.007887479305888215, "clip_ratio/low_mean": 0.000994703887651364, "clip_ratio/low_min": 0.000994703887651364, "clip_ratio/region_mean": 0.008882183115929365, "completions/clipped_ratio": 0.0, "completions/max_length": 2359.0, "completions/max_terminated_length": 2359.0, "completions/mean_length": 1282.52783203125, "completions/mean_terminated_length": 1282.52783203125, "completions/min_length": 561.0, "completions/min_terminated_length": 561.0, "entropy": 0.2848661094903946, "epoch": 0.019658119658119658, "frac_reward_zero_std": 0.0, "grad_norm": 0.051431454718112946, "learning_rate": 5e-05, "loss": -0.0024, "num_tokens": 3318716.0, "reward": 0.37804630398750305, "reward_std": 0.3656369149684906, "rewards/hint_following/mean": 0.8055555820465088, "rewards/hint_following/std": 0.4013865292072296, "rewards/length_penalty/mean": -0.42750924825668335, "rewards/length_penalty/std": 0.16107404232025146, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9896759986877441, "sampling/importance_sampling_ratio/min": 0.09941968321800232, "sampling/sampling_logp_difference/max": 2.3084051609039307, "sampling/sampling_logp_difference/mean": 0.029690390452742577, "step": 46, "step_time": 41.876618986018 }, { "clip_ratio/high_max": 0.003261159232351929, "clip_ratio/high_mean": 0.003261159232351929, "clip_ratio/low_mean": 0.0030720558910009763, "clip_ratio/low_min": 0.0030720558910009763, "clip_ratio/region_mean": 0.0063332150457426906, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 3000.0, "completions/max_terminated_length": 2990.0, "completions/mean_length": 1868.77783203125, "completions/mean_terminated_length": 1765.939453125, "completions/min_length": 947.0, "completions/min_terminated_length": 947.0, "entropy": 0.3000795841217041, "epoch": 0.020085470085470087, "frac_reward_zero_std": 0.0, "grad_norm": 0.07183089107275009, "learning_rate": 5e-05, "loss": 0.0154, "num_tokens": 3395160.0, "reward": -0.12292594462633133, "reward_std": 0.785350501537323, "rewards/hint_following/mean": 0.5, "rewards/hint_following/std": 0.6546536684036255, "rewards/length_penalty/mean": -0.6229259371757507, "rewards/length_penalty/std": 0.2120964676141739, "sampling/importance_sampling_ratio/max": 2.9097461700439453, "sampling/importance_sampling_ratio/mean": 0.988696277141571, "sampling/importance_sampling_ratio/min": 0.10777360945940018, "sampling/sampling_logp_difference/max": 2.227722406387329, "sampling/sampling_logp_difference/mean": 0.030243437737226486, "step": 47, "step_time": 58.67247669992503 }, { "clip_ratio/high_max": 0.008134961283455292, "clip_ratio/high_mean": 0.008134961283455292, "clip_ratio/low_mean": 0.0004609357565641403, "clip_ratio/low_min": 0.0004609357565641403, "clip_ratio/region_mean": 0.008595897040019432, "completions/clipped_ratio": 0.2222222238779068, "completions/max_length": 3000.0, "completions/max_terminated_length": 2543.0, "completions/mean_length": 1645.8333740234375, "completions/mean_terminated_length": 1258.9285888671875, "completions/min_length": 729.0, "completions/min_terminated_length": 729.0, "entropy": 0.24671275417009988, "epoch": 0.020512820512820513, "frac_reward_zero_std": 0.0, "grad_norm": 0.06476566940546036, "learning_rate": 5e-05, "loss": 0.0513, "num_tokens": 3461970.0, "reward": 0.006944431457668543, "reward_std": 1.0957543849945068, "rewards/hint_following/mean": 0.5555555820465088, "rewards/hint_following/std": 0.8432740569114685, "rewards/length_penalty/mean": -0.5486111044883728, "rewards/length_penalty/std": 0.2772509455680847, "sampling/importance_sampling_ratio/max": 2.0642147064208984, "sampling/importance_sampling_ratio/mean": 0.9907264113426208, "sampling/importance_sampling_ratio/min": 0.04710150137543678, "sampling/sampling_logp_difference/max": 3.055450439453125, "sampling/sampling_logp_difference/mean": 0.025603190064430237, "step": 48, "step_time": 55.27118606586009 }, { "clip_ratio/high_max": 0.0011131278927981232, "clip_ratio/high_mean": 0.0011131278927981232, "clip_ratio/low_mean": 0.006329516724993785, "clip_ratio/low_min": 0.006329516724993785, "clip_ratio/region_mean": 0.007442644564434886, "completions/clipped_ratio": 0.0, "completions/max_length": 1867.0, "completions/max_terminated_length": 1867.0, "completions/mean_length": 1157.6666259765625, "completions/mean_terminated_length": 1157.6666259765625, "completions/min_length": 722.0, "completions/min_terminated_length": 722.0, "entropy": 0.2628251537680626, "epoch": 0.02094017094017094, "frac_reward_zero_std": 0.0, "grad_norm": 0.06659235060214996, "learning_rate": 5e-05, "loss": 0.0356, "num_tokens": 3510510.0, "reward": 0.5863333344459534, "reward_std": 0.22435788810253143, "rewards/hint_following/mean": 0.9722222089767456, "rewards/hint_following/std": 0.1666666716337204, "rewards/length_penalty/mean": -0.3858889043331146, "rewards/length_penalty/std": 0.10318709164857864, "sampling/importance_sampling_ratio/max": 2.3799936771392822, "sampling/importance_sampling_ratio/mean": 0.9904940128326416, "sampling/importance_sampling_ratio/min": 0.06988711655139923, "sampling/sampling_logp_difference/max": 2.6608738899230957, "sampling/sampling_logp_difference/mean": 0.030541572719812393, "step": 49, "step_time": 33.86789354507346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 3000.0, "completions/max_terminated_length": 2940.0, "completions/mean_length": 1683.0, "completions/mean_terminated_length": 1419.60009765625, "completions/min_length": 730.0, "completions/min_terminated_length": 730.0, "entropy": 0.24505582451820374, "epoch": 0.021367521367521368, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 3577332.0, "reward": 0.10566666722297668, "reward_std": 0.9825097322463989, "rewards/hint_following/mean": 0.6666666865348816, "rewards/hint_following/std": 0.7559289932250977, "rewards/length_penalty/mean": -0.5609999895095825, "rewards/length_penalty/std": 0.30474284291267395, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9918250441551208, "sampling/importance_sampling_ratio/min": 0.06838197261095047, "sampling/sampling_logp_difference/max": 2.6826460361480713, "sampling/sampling_logp_difference/mean": 0.028820084407925606, "step": 50, "step_time": 54.36680988012813 } ], "logging_steps": 1, "max_steps": 250, "num_input_tokens_seen": 3577332, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 6, "trial_name": null, "trial_params": null }