diff --git "a/checkpoint-26333/trainer_state.json" "b/checkpoint-26333/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-26333/trainer_state.json" @@ -0,0 +1,34237 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 26333, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025555557012557982, + "completions/max_length": 476.9, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 188.67223256429037, + "completions/mean_terminated_length": 180.62073872884113, + "completions/min_length": 44.06666666666667, + "completions/min_terminated_length": 44.06666666666667, + "entropy": 0.19599945321679116, + "epoch": 0.0011392549272775604, + "eval/gt_acc_batch": 0.7266666869322459, + "frac_reward_zero_std": 1.0, + "grad_norm": 41.69678497314453, + "kd/policy_loss": -0.140931467252085, + "kd/rkl_advantage_mean": 1.2623825818921128, + "kd/rkl_advantage_p95": 3.9140202621618907, + "kd/rkl_advantage_std": 1.5576938132445017, + "kd/ssd_loss": 0.0, + "learning_rate": 9.98898720236965e-07, + "loss": -0.5637259165445964, + "num_tokens": 979556.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7266666710376739, + "rewards/gt_logging_reward/std": 0.4374186098575592, + "sampling/importance_sampling_ratio/max": 1.899801218509674, + "sampling/importance_sampling_ratio/mean": 0.9982677181561788, + "sampling/importance_sampling_ratio/min": 0.4686829000711441, + "sampling/sampling_logp_difference/max": 0.2666013439496358, + "sampling/sampling_logp_difference/mean": 0.004447124774257342, + "step": 30, + "step_time": 7.83714348602031, + "student/entropy": 0.19599945321679116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029166668436179558, + "completions/max_length": 505.43333333333334, + "completions/max_terminated_length": 469.1333333333333, + "completions/mean_length": 196.5341766357422, + "completions/mean_terminated_length": 187.06739298502603, + "completions/min_length": 48.233333333333334, + "completions/min_terminated_length": 48.233333333333334, + "entropy": 0.19372152332216502, + "epoch": 0.002278509854555121, + "eval/gt_acc_batch": 0.7163889169692993, + "frac_reward_zero_std": 1.0, + "grad_norm": 55.56889343261719, + "kd/policy_loss": -0.7193360067903996, + "kd/rkl_advantage_mean": 4.391698981324832, + "kd/rkl_advantage_p95": 7.800063109397888, + "kd/rkl_advantage_std": 2.257473360498746, + "kd/ssd_loss": 0.0, + "learning_rate": 9.977594653096876e-07, + "loss": -2.877344512939453, + "num_tokens": 1986711.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7163888931274414, + "rewards/gt_logging_reward/std": 0.44609825909137724, + "sampling/importance_sampling_ratio/max": 1.9319464246431985, + "sampling/importance_sampling_ratio/mean": 1.003615967432658, + "sampling/importance_sampling_ratio/min": 0.45573407113552095, + "sampling/sampling_logp_difference/max": 0.2922445158163706, + "sampling/sampling_logp_difference/mean": 0.004456543449002008, + "step": 60, + "step_time": 7.983978455591326, + "student/entropy": 0.19372152332216502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023055557037393253, + "completions/max_length": 497.56666666666666, + "completions/max_terminated_length": 463.9, + "completions/mean_length": 193.20001017252605, + "completions/mean_terminated_length": 185.67325744628906, + "completions/min_length": 54.03333333333333, + "completions/min_terminated_length": 54.03333333333333, + "entropy": 0.1798501955345273, + "epoch": 0.0034177647818326813, + "eval/gt_acc_batch": 0.7258333464463552, + "frac_reward_zero_std": 1.0, + "grad_norm": 78.53369903564453, + "kd/policy_loss": -1.513459900021553, + "kd/rkl_advantage_mean": 8.898690394560496, + "kd/rkl_advantage_p95": 14.606252932548523, + "kd/rkl_advantage_std": 4.007087437311808, + "kd/ssd_loss": 0.0, + "learning_rate": 9.9662021038241e-07, + "loss": -6.053840128580729, + "num_tokens": 2987111.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7258333325386047, + "rewards/gt_logging_reward/std": 0.4413954029480616, + "sampling/importance_sampling_ratio/max": 1.954199461142222, + "sampling/importance_sampling_ratio/mean": 0.995514287551244, + "sampling/importance_sampling_ratio/min": 0.4436719059944153, + "sampling/sampling_logp_difference/max": 0.2707558572292328, + "sampling/sampling_logp_difference/mean": 0.0043091895213971535, + "step": 90, + "step_time": 7.845766410064728, + "student/entropy": 0.1798501955345273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021944445899377267, + "completions/max_length": 484.93333333333334, + "completions/max_terminated_length": 441.76666666666665, + "completions/mean_length": 184.85556640625, + "completions/mean_terminated_length": 177.6335205078125, + "completions/min_length": 52.2, + "completions/min_terminated_length": 52.2, + "entropy": 0.17184054907411336, + "epoch": 0.004557019709110242, + "eval/gt_acc_batch": 0.7597222407658895, + "frac_reward_zero_std": 1.0, + "grad_norm": 95.32417297363281, + "kd/policy_loss": -2.716463388999303, + "kd/rkl_advantage_mean": 15.562823748588562, + "kd/rkl_advantage_p95": 24.098573923110962, + "kd/rkl_advantage_std": 6.491960348685582, + "kd/ssd_loss": 0.0, + "learning_rate": 9.954809554551323e-07, + "loss": -10.865855916341145, + "num_tokens": 3962951.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7597222228844961, + "rewards/gt_logging_reward/std": 0.418232761323452, + "sampling/importance_sampling_ratio/max": 1.9486569007237753, + "sampling/importance_sampling_ratio/mean": 1.0014465709527334, + "sampling/importance_sampling_ratio/min": 0.4404404371976852, + "sampling/sampling_logp_difference/max": 0.2967432816823324, + "sampling/sampling_logp_difference/mean": 0.004454438737593591, + "step": 120, + "step_time": 7.9130238673028845, + "student/entropy": 0.17184054907411336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013611111820985873, + "completions/max_length": 468.0, + "completions/max_terminated_length": 453.6, + "completions/mean_length": 184.43028767903647, + "completions/mean_terminated_length": 179.9935760498047, + "completions/min_length": 49.96666666666667, + "completions/min_terminated_length": 49.96666666666667, + "entropy": 0.1684624078993996, + "epoch": 0.005696274636387802, + "eval/gt_acc_batch": 0.7791666984558105, + "frac_reward_zero_std": 1.0, + "grad_norm": 104.59193420410156, + "kd/policy_loss": -4.2013640721639, + "kd/rkl_advantage_mean": 23.82131028175354, + "kd/rkl_advantage_p95": 35.23421998023987, + "kd/rkl_advantage_std": 9.378543941179911, + "kd/ssd_loss": 0.0, + "learning_rate": 9.943417005278546e-07, + "loss": -16.805455525716145, + "num_tokens": 4928332.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7791666626930237, + "rewards/gt_logging_reward/std": 0.40874949196974436, + "sampling/importance_sampling_ratio/max": 1.8500961701075236, + "sampling/importance_sampling_ratio/mean": 0.9883681436379751, + "sampling/importance_sampling_ratio/min": 0.40977474947770437, + "sampling/sampling_logp_difference/max": 0.354287858804067, + "sampling/sampling_logp_difference/mean": 0.00448098941706121, + "step": 150, + "step_time": 7.659808489082692, + "student/entropy": 0.1684624078993996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018055556838711104, + "completions/max_length": 493.3, + "completions/max_terminated_length": 462.6666666666667, + "completions/mean_length": 188.9019541422526, + "completions/mean_terminated_length": 183.04210764567057, + "completions/min_length": 50.63333333333333, + "completions/min_terminated_length": 50.63333333333333, + "entropy": 0.1540275743852059, + "epoch": 0.0068355295636653626, + "eval/gt_acc_batch": 0.7663889189561208, + "frac_reward_zero_std": 1.0, + "grad_norm": 125.55032348632812, + "kd/policy_loss": -6.022807987531026, + "kd/rkl_advantage_mean": 33.80864222844442, + "kd/rkl_advantage_p95": 47.755531533559164, + "kd/rkl_advantage_std": 12.79307312965393, + "kd/ssd_loss": 0.0, + "learning_rate": 9.932024456005772e-07, + "loss": -24.09123331705729, + "num_tokens": 5908627.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7663888891537984, + "rewards/gt_logging_reward/std": 0.4162752379973729, + "sampling/importance_sampling_ratio/max": 1.8868409276008606, + "sampling/importance_sampling_ratio/mean": 0.9996463318665822, + "sampling/importance_sampling_ratio/min": 0.42902452647686007, + "sampling/sampling_logp_difference/max": 0.37517107725143434, + "sampling/sampling_logp_difference/mean": 0.004343474733953675, + "step": 180, + "step_time": 7.822546921945953, + "student/entropy": 0.1540275743852059 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017222223399827877, + "completions/max_length": 473.03333333333336, + "completions/max_terminated_length": 445.0, + "completions/mean_length": 186.02862141927082, + "completions/mean_terminated_length": 180.40457458496093, + "completions/min_length": 55.5, + "completions/min_terminated_length": 55.5, + "entropy": 0.15306980032473802, + "epoch": 0.007974784490942924, + "eval/gt_acc_batch": 0.7600000162919363, + "frac_reward_zero_std": 1.0, + "grad_norm": 117.56169128417969, + "kd/policy_loss": -7.377011565367381, + "kd/rkl_advantage_mean": 42.247389237085976, + "kd/rkl_advantage_p95": 59.225398031870526, + "kd/rkl_advantage_std": 16.71758441925049, + "kd/ssd_loss": 0.0, + "learning_rate": 9.920631906732996e-07, + "loss": -29.50804443359375, + "num_tokens": 6875746.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7600000003973643, + "rewards/gt_logging_reward/std": 0.42220642666021985, + "sampling/importance_sampling_ratio/max": 1.8900311907132468, + "sampling/importance_sampling_ratio/mean": 0.993595149119695, + "sampling/importance_sampling_ratio/min": 0.4206998775402705, + "sampling/sampling_logp_difference/max": 0.3918265521526337, + "sampling/sampling_logp_difference/mean": 0.004424939560703933, + "step": 210, + "step_time": 7.618048585268358, + "student/entropy": 0.15306980032473802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018333334289491177, + "completions/max_length": 483.1, + "completions/max_terminated_length": 446.46666666666664, + "completions/mean_length": 184.75306447347006, + "completions/mean_terminated_length": 178.87464497884113, + "completions/min_length": 49.5, + "completions/min_terminated_length": 49.5, + "entropy": 0.14876011765251557, + "epoch": 0.009114039418220483, + "eval/gt_acc_batch": 0.7602777977784475, + "frac_reward_zero_std": 1.0, + "grad_norm": 96.66764068603516, + "kd/policy_loss": -9.308236944675446, + "kd/rkl_advantage_mean": 51.7335693359375, + "kd/rkl_advantage_p95": 69.57423842748007, + "kd/rkl_advantage_std": 19.047987071673074, + "kd/ssd_loss": 0.0, + "learning_rate": 9.909239357460222e-07, + "loss": -37.232950846354164, + "num_tokens": 7833809.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.760277779897054, + "rewards/gt_logging_reward/std": 0.4215780129035314, + "sampling/importance_sampling_ratio/max": 1.9830000917116801, + "sampling/importance_sampling_ratio/mean": 1.0025597075621286, + "sampling/importance_sampling_ratio/min": 0.4144452194372813, + "sampling/sampling_logp_difference/max": 0.47801239490509034, + "sampling/sampling_logp_difference/mean": 0.004535982113641997, + "step": 240, + "step_time": 7.694364592495064, + "student/entropy": 0.14876011765251557 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026111112721264363, + "completions/max_length": 492.3666666666667, + "completions/max_terminated_length": 457.3666666666667, + "completions/mean_length": 189.97639923095704, + "completions/mean_terminated_length": 181.52837219238282, + "completions/min_length": 52.666666666666664, + "completions/min_terminated_length": 52.666666666666664, + "entropy": 0.1518039566775163, + "epoch": 0.010253294345498045, + "eval/gt_acc_batch": 0.7527777930100759, + "frac_reward_zero_std": 1.0, + "grad_norm": 120.29234313964844, + "kd/policy_loss": -10.697002009550731, + "kd/rkl_advantage_mean": 59.42068030039469, + "kd/rkl_advantage_p95": 78.30765673319499, + "kd/rkl_advantage_std": 21.66599412759145, + "kd/ssd_loss": 0.0, + "learning_rate": 9.897846808187445e-07, + "loss": -42.7880126953125, + "num_tokens": 8820828.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.752777777115504, + "rewards/gt_logging_reward/std": 0.41847467521826426, + "sampling/importance_sampling_ratio/max": 2.010831268628438, + "sampling/importance_sampling_ratio/mean": 1.006870722770691, + "sampling/importance_sampling_ratio/min": 0.4128865472972393, + "sampling/sampling_logp_difference/max": 0.5652268946170806, + "sampling/sampling_logp_difference/mean": 0.004605383200881382, + "step": 270, + "step_time": 7.854774193613169, + "student/entropy": 0.1518039566775163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013611111789941788, + "completions/max_length": 468.3666666666667, + "completions/max_terminated_length": 432.06666666666666, + "completions/mean_length": 173.38445409138998, + "completions/mean_terminated_length": 168.79506963094076, + "completions/min_length": 49.6, + "completions/min_terminated_length": 49.6, + "entropy": 0.14717570319771767, + "epoch": 0.011392549272775604, + "eval/gt_acc_batch": 0.7708333452542623, + "frac_reward_zero_std": 1.0, + "grad_norm": 105.3351058959961, + "kd/policy_loss": -11.895396637916566, + "kd/rkl_advantage_mean": 66.56356236139933, + "kd/rkl_advantage_p95": 86.43890908559163, + "kd/rkl_advantage_std": 23.732891567548116, + "kd/ssd_loss": 0.0, + "learning_rate": 9.88645425891467e-07, + "loss": -47.581591796875, + "num_tokens": 9740196.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7708333373069763, + "rewards/gt_logging_reward/std": 0.41020310719807945, + "sampling/importance_sampling_ratio/max": 2.051044499874115, + "sampling/importance_sampling_ratio/mean": 1.0032211899757386, + "sampling/importance_sampling_ratio/min": 0.4047176147500674, + "sampling/sampling_logp_difference/max": 0.5486619393030803, + "sampling/sampling_logp_difference/mean": 0.004586749570444226, + "step": 300, + "step_time": 7.595890953737156, + "student/entropy": 0.14717570319771767 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024722223511586587, + "completions/max_length": 498.03333333333336, + "completions/max_terminated_length": 461.3333333333333, + "completions/mean_length": 191.10306396484376, + "completions/mean_terminated_length": 183.02750193277996, + "completions/min_length": 46.8, + "completions/min_terminated_length": 46.8, + "entropy": 0.15626121852546931, + "epoch": 0.012531804200053166, + "eval/gt_acc_batch": 0.7455555816491445, + "frac_reward_zero_std": 1.0, + "grad_norm": 82.22247314453125, + "kd/policy_loss": -12.888094385464987, + "kd/rkl_advantage_mean": 72.38278713226319, + "kd/rkl_advantage_p95": 94.1824119567871, + "kd/rkl_advantage_std": 26.302427236239115, + "kd/ssd_loss": 0.0, + "learning_rate": 9.875061709641895e-07, + "loss": -51.55237630208333, + "num_tokens": 10735471.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7455555538336436, + "rewards/gt_logging_reward/std": 0.428698476155599, + "sampling/importance_sampling_ratio/max": 2.0161656300226847, + "sampling/importance_sampling_ratio/mean": 0.9961058259010315, + "sampling/importance_sampling_ratio/min": 0.37376941194136937, + "sampling/sampling_logp_difference/max": 0.6893646240234375, + "sampling/sampling_logp_difference/mean": 0.004744267680992683, + "step": 330, + "step_time": 7.99581612344676, + "student/entropy": 0.15626121852546931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015555556459973256, + "completions/max_length": 472.6666666666667, + "completions/max_terminated_length": 439.7, + "completions/mean_length": 183.30528717041017, + "completions/mean_terminated_length": 178.1916269938151, + "completions/min_length": 53.93333333333333, + "completions/min_terminated_length": 53.93333333333333, + "entropy": 0.13420339232931536, + "epoch": 0.013671059127330725, + "eval/gt_acc_batch": 0.7677777886390686, + "frac_reward_zero_std": 1.0, + "grad_norm": 93.04169464111328, + "kd/policy_loss": -13.95430630048116, + "kd/rkl_advantage_mean": 77.81620346705118, + "kd/rkl_advantage_p95": 102.02783107757568, + "kd/rkl_advantage_std": 28.77157617410024, + "kd/ssd_loss": 0.0, + "learning_rate": 9.863669160369118e-07, + "loss": -55.81721598307292, + "num_tokens": 11707970.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7677777747313181, + "rewards/gt_logging_reward/std": 0.41553530693054197, + "sampling/importance_sampling_ratio/max": 1.9252396782239278, + "sampling/importance_sampling_ratio/mean": 1.0042305827140807, + "sampling/importance_sampling_ratio/min": 0.39444501089553036, + "sampling/sampling_logp_difference/max": 0.6446903924147288, + "sampling/sampling_logp_difference/mean": 0.004362575880562266, + "step": 360, + "step_time": 7.795245896675624, + "student/entropy": 0.13420339232931536 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444445812453825, + "completions/max_length": 478.3, + "completions/max_terminated_length": 440.9, + "completions/mean_length": 183.11278686523437, + "completions/mean_terminated_length": 175.03037821451824, + "completions/min_length": 50.1, + "completions/min_terminated_length": 50.1, + "entropy": 0.14449129470934471, + "epoch": 0.014810314054608286, + "eval/gt_acc_batch": 0.744444473584493, + "frac_reward_zero_std": 1.0, + "grad_norm": 80.87189483642578, + "kd/policy_loss": -15.283781901995342, + "kd/rkl_advantage_mean": 85.92365067799886, + "kd/rkl_advantage_p95": 110.78063570658365, + "kd/rkl_advantage_std": 31.35953189531962, + "kd/ssd_loss": 0.0, + "learning_rate": 9.852276611096342e-07, + "loss": -61.1351318359375, + "num_tokens": 12667024.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7444444398085276, + "rewards/gt_logging_reward/std": 0.43154789706071217, + "sampling/importance_sampling_ratio/max": 2.01114209095637, + "sampling/importance_sampling_ratio/mean": 1.0006886879603067, + "sampling/importance_sampling_ratio/min": 0.36882616952061653, + "sampling/sampling_logp_difference/max": 0.7083962301413218, + "sampling/sampling_logp_difference/mean": 0.0046289332831899324, + "step": 390, + "step_time": 7.644774106979215, + "student/entropy": 0.14449129470934471 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.011111111690600714, + "completions/max_length": 487.23333333333335, + "completions/max_terminated_length": 454.6333333333333, + "completions/mean_length": 179.42528737386067, + "completions/mean_terminated_length": 175.82404073079428, + "completions/min_length": 53.06666666666667, + "completions/min_terminated_length": 53.06666666666667, + "entropy": 0.13817637773851554, + "epoch": 0.015949568981885848, + "eval/gt_acc_batch": 0.7888889173666637, + "frac_reward_zero_std": 1.0, + "grad_norm": 72.26576232910156, + "kd/policy_loss": -16.695345362027485, + "kd/rkl_advantage_mean": 92.27905680338542, + "kd/rkl_advantage_p95": 117.1095771153768, + "kd/rkl_advantage_std": 31.460352643330893, + "kd/ssd_loss": 0.0, + "learning_rate": 9.840884061823566e-07, + "loss": -66.78139241536458, + "num_tokens": 13607131.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7888888915379842, + "rewards/gt_logging_reward/std": 0.39881707429885865, + "sampling/importance_sampling_ratio/max": 1.8206988096237182, + "sampling/importance_sampling_ratio/mean": 0.992443323135376, + "sampling/importance_sampling_ratio/min": 0.42406642039616904, + "sampling/sampling_logp_difference/max": 0.557313722372055, + "sampling/sampling_logp_difference/mean": 0.004471641778945923, + "step": 420, + "step_time": 7.6503381504444405, + "student/entropy": 0.13817637773851554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019722223468124866, + "completions/max_length": 478.3, + "completions/max_terminated_length": 441.3333333333333, + "completions/mean_length": 183.17445500691733, + "completions/mean_terminated_length": 176.67584355672201, + "completions/min_length": 46.06666666666667, + "completions/min_terminated_length": 46.06666666666667, + "entropy": 0.13939987079550822, + "epoch": 0.017088823909163407, + "eval/gt_acc_batch": 0.7433333555857341, + "frac_reward_zero_std": 1.0, + "grad_norm": 71.77312469482422, + "kd/policy_loss": -16.51058549086253, + "kd/rkl_advantage_mean": 93.99352588653565, + "kd/rkl_advantage_p95": 122.20874722798665, + "kd/rkl_advantage_std": 35.313513898849486, + "kd/ssd_loss": 0.0, + "learning_rate": 9.829491512550791e-07, + "loss": -66.04234619140625, + "num_tokens": 14574591.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7433333337306977, + "rewards/gt_logging_reward/std": 0.42900978326797484, + "sampling/importance_sampling_ratio/max": 1.995363998413086, + "sampling/importance_sampling_ratio/mean": 0.9971106112003326, + "sampling/importance_sampling_ratio/min": 0.4244975219170252, + "sampling/sampling_logp_difference/max": 0.5669575035572052, + "sampling/sampling_logp_difference/mean": 0.004523365362547338, + "step": 450, + "step_time": 7.820953815515774, + "student/entropy": 0.13939987079550822 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018611112236976625, + "completions/max_length": 474.1, + "completions/max_terminated_length": 437.23333333333335, + "completions/mean_length": 185.2805643717448, + "completions/mean_terminated_length": 179.3388463338216, + "completions/min_length": 54.86666666666667, + "completions/min_terminated_length": 54.86666666666667, + "entropy": 0.13804754273345074, + "epoch": 0.018228078836440967, + "eval/gt_acc_batch": 0.7586111346880595, + "frac_reward_zero_std": 1.0, + "grad_norm": 68.87535095214844, + "kd/policy_loss": -17.53249555428823, + "kd/rkl_advantage_mean": 98.34410928090414, + "kd/rkl_advantage_p95": 126.52606716156006, + "kd/rkl_advantage_std": 35.73711473941803, + "kd/ssd_loss": 0.0, + "learning_rate": 9.818098963278015e-07, + "loss": -70.12998046875, + "num_tokens": 15542361.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7586111108462016, + "rewards/gt_logging_reward/std": 0.41951424777507784, + "sampling/importance_sampling_ratio/max": 1.9949932813644409, + "sampling/importance_sampling_ratio/mean": 0.9948066135247549, + "sampling/importance_sampling_ratio/min": 0.3423141730328401, + "sampling/sampling_logp_difference/max": 0.7058973809083303, + "sampling/sampling_logp_difference/mean": 0.004521623610829314, + "step": 480, + "step_time": 7.749692405213136, + "student/entropy": 0.13804754273345074 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01916666782150666, + "completions/max_length": 477.96666666666664, + "completions/max_terminated_length": 445.7, + "completions/mean_length": 178.68250834147136, + "completions/mean_terminated_length": 172.28610382080078, + "completions/min_length": 40.833333333333336, + "completions/min_terminated_length": 40.833333333333336, + "entropy": 0.1407859311128656, + "epoch": 0.019367333763718526, + "eval/gt_acc_batch": 0.7647222359975179, + "frac_reward_zero_std": 1.0, + "grad_norm": 55.42365646362305, + "kd/policy_loss": -17.57277130285899, + "kd/rkl_advantage_mean": 101.13098894755045, + "kd/rkl_advantage_p95": 129.96323884328206, + "kd/rkl_advantage_std": 37.900245650609335, + "kd/ssd_loss": 0.0, + "learning_rate": 9.80670641400524e-07, + "loss": -70.29108072916667, + "num_tokens": 16488106.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.764722224076589, + "rewards/gt_logging_reward/std": 0.4158546507358551, + "sampling/importance_sampling_ratio/max": 1.8697431206703186, + "sampling/importance_sampling_ratio/mean": 0.9913185278574626, + "sampling/importance_sampling_ratio/min": 0.3391820423305035, + "sampling/sampling_logp_difference/max": 0.8182136336962382, + "sampling/sampling_logp_difference/mean": 0.0045899656834080815, + "step": 510, + "step_time": 7.832733157979479, + "student/entropy": 0.1407859311128656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.014166667747000853, + "completions/max_length": 468.5, + "completions/max_terminated_length": 421.93333333333334, + "completions/mean_length": 170.7880661010742, + "completions/mean_terminated_length": 166.03417053222657, + "completions/min_length": 41.06666666666667, + "completions/min_terminated_length": 41.06666666666667, + "entropy": 0.13352700894077618, + "epoch": 0.02050658869099609, + "eval/gt_acc_batch": 0.7911111275355022, + "frac_reward_zero_std": 1.0, + "grad_norm": 70.99797058105469, + "kd/policy_loss": -18.862686467170715, + "kd/rkl_advantage_mean": 105.80604311625163, + "kd/rkl_advantage_p95": 133.37760378519695, + "kd/rkl_advantage_std": 36.92920085589091, + "kd/ssd_loss": 0.0, + "learning_rate": 9.795313864732464e-07, + "loss": -75.4507568359375, + "num_tokens": 17407263.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7911111096541087, + "rewards/gt_logging_reward/std": 0.3947017103433609, + "sampling/importance_sampling_ratio/max": 1.9447731137275697, + "sampling/importance_sampling_ratio/mean": 0.9966152608394623, + "sampling/importance_sampling_ratio/min": 0.37790223558743796, + "sampling/sampling_logp_difference/max": 0.7160782476266225, + "sampling/sampling_logp_difference/mean": 0.0043746281027173, + "step": 540, + "step_time": 7.730104634654708, + "student/entropy": 0.13352700894077618 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.011111111814777056, + "completions/max_length": 447.53333333333336, + "completions/max_terminated_length": 417.0, + "completions/mean_length": 172.06834259033204, + "completions/mean_terminated_length": 168.3518793741862, + "completions/min_length": 46.166666666666664, + "completions/min_terminated_length": 46.166666666666664, + "entropy": 0.13039307352155446, + "epoch": 0.02164584361827365, + "eval/gt_acc_batch": 0.7866666873296102, + "frac_reward_zero_std": 1.0, + "grad_norm": 63.10587692260742, + "kd/policy_loss": -18.73554105758667, + "kd/rkl_advantage_mean": 105.5212023417155, + "kd/rkl_advantage_p95": 136.05569178263346, + "kd/rkl_advantage_std": 38.629692538579306, + "kd/ssd_loss": 0.0, + "learning_rate": 9.783921315459688e-07, + "loss": -74.94217122395834, + "num_tokens": 18327213.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7866666714350382, + "rewards/gt_logging_reward/std": 0.402436929444472, + "sampling/importance_sampling_ratio/max": 1.9135347565015157, + "sampling/importance_sampling_ratio/mean": 0.9944364468256632, + "sampling/importance_sampling_ratio/min": 0.34644420047601066, + "sampling/sampling_logp_difference/max": 0.7387901763121287, + "sampling/sampling_logp_difference/mean": 0.00422365894385924, + "step": 570, + "step_time": 7.450159085142271, + "student/entropy": 0.13039307352155446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016111111951371035, + "completions/max_length": 452.43333333333334, + "completions/max_terminated_length": 424.03333333333336, + "completions/mean_length": 174.7697326660156, + "completions/mean_terminated_length": 169.32348073323567, + "completions/min_length": 44.13333333333333, + "completions/min_terminated_length": 44.13333333333333, + "entropy": 0.13229780768354735, + "epoch": 0.02278509854555121, + "eval/gt_acc_batch": 0.7525000135103862, + "frac_reward_zero_std": 1.0, + "grad_norm": 49.473106384277344, + "kd/policy_loss": -19.06785488128662, + "kd/rkl_advantage_mean": 107.10229918162028, + "kd/rkl_advantage_p95": 138.60916061401366, + "kd/rkl_advantage_std": 39.92631467183431, + "kd/ssd_loss": 0.0, + "learning_rate": 9.772528766186914e-07, + "loss": -76.27141927083333, + "num_tokens": 19259984.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7524999996026357, + "rewards/gt_logging_reward/std": 0.4208496044079463, + "sampling/importance_sampling_ratio/max": 2.0110241293907167, + "sampling/importance_sampling_ratio/mean": 0.9958545744419098, + "sampling/importance_sampling_ratio/min": 0.3518367201089859, + "sampling/sampling_logp_difference/max": 0.9247934063275655, + "sampling/sampling_logp_difference/mean": 0.004344920356137057, + "step": 600, + "step_time": 7.619482866216761, + "student/entropy": 0.13229780768354735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01055555623024702, + "completions/max_length": 475.0, + "completions/max_terminated_length": 444.8666666666667, + "completions/mean_length": 174.15028610229493, + "completions/mean_terminated_length": 170.6033622741699, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.13310089310010273, + "epoch": 0.02392435347282877, + "eval/gt_acc_batch": 0.7408333520094553, + "frac_reward_zero_std": 1.0, + "grad_norm": 64.99547576904297, + "kd/policy_loss": -20.032616662979127, + "kd/rkl_advantage_mean": 111.12264620463053, + "kd/rkl_advantage_p95": 140.83590253194174, + "kd/rkl_advantage_std": 39.08147749900818, + "kd/ssd_loss": 0.0, + "learning_rate": 9.761136216914138e-07, + "loss": -80.13046875, + "num_tokens": 20185685.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.740833330154419, + "rewards/gt_logging_reward/std": 0.43316762149333954, + "sampling/importance_sampling_ratio/max": 2.0341086188952127, + "sampling/importance_sampling_ratio/mean": 1.002538683017095, + "sampling/importance_sampling_ratio/min": 0.38289637813965477, + "sampling/sampling_logp_difference/max": 0.7324491381645203, + "sampling/sampling_logp_difference/mean": 0.00431928769685328, + "step": 630, + "step_time": 7.658903203423446, + "student/entropy": 0.13310089310010273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01750000100582838, + "completions/max_length": 479.96666666666664, + "completions/max_terminated_length": 435.5, + "completions/mean_length": 178.75250803629558, + "completions/mean_terminated_length": 172.80643412272136, + "completions/min_length": 44.63333333333333, + "completions/min_terminated_length": 44.63333333333333, + "entropy": 0.13295599209765593, + "epoch": 0.02506360840010633, + "eval/gt_acc_batch": 0.710833348830541, + "frac_reward_zero_std": 1.0, + "grad_norm": 72.1101303100586, + "kd/policy_loss": -20.949782729148865, + "kd/rkl_advantage_mean": 114.7668446222941, + "kd/rkl_advantage_p95": 142.90474433898925, + "kd/rkl_advantage_std": 38.965674209594724, + "kd/ssd_loss": 0.0, + "learning_rate": 9.749743667641363e-07, + "loss": -83.79912109375, + "num_tokens": 21136402.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.710833336909612, + "rewards/gt_logging_reward/std": 0.44462570548057556, + "sampling/importance_sampling_ratio/max": 2.178599766890208, + "sampling/importance_sampling_ratio/mean": 1.006586492061615, + "sampling/importance_sampling_ratio/min": 0.40391783267259596, + "sampling/sampling_logp_difference/max": 0.7610865533351898, + "sampling/sampling_logp_difference/mean": 0.004358640817614893, + "step": 660, + "step_time": 7.822033193633737, + "student/entropy": 0.13295599209765593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015277778667708238, + "completions/max_length": 476.3666666666667, + "completions/max_terminated_length": 446.73333333333335, + "completions/mean_length": 179.1533426920573, + "completions/mean_terminated_length": 173.97613881429035, + "completions/min_length": 45.3, + "completions/min_terminated_length": 45.3, + "entropy": 0.13110877859095732, + "epoch": 0.02620286332738389, + "eval/gt_acc_batch": 0.7300000250339508, + "frac_reward_zero_std": 1.0, + "grad_norm": 63.45765686035156, + "kd/policy_loss": -19.895428188641866, + "kd/rkl_advantage_mean": 112.38459695180258, + "kd/rkl_advantage_p95": 144.50572026570637, + "kd/rkl_advantage_std": 41.743690458933514, + "kd/ssd_loss": 0.0, + "learning_rate": 9.738351118368587e-07, + "loss": -79.58170572916667, + "num_tokens": 22096866.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7300000011920929, + "rewards/gt_logging_reward/std": 0.43528696298599245, + "sampling/importance_sampling_ratio/max": 2.1605713526407877, + "sampling/importance_sampling_ratio/mean": 0.9977336049079895, + "sampling/importance_sampling_ratio/min": 0.364988507827123, + "sampling/sampling_logp_difference/max": 0.9214066922664642, + "sampling/sampling_logp_difference/mean": 0.0043837680015712975, + "step": 690, + "step_time": 7.954079896754895, + "student/entropy": 0.13110877859095732 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.010277778437982003, + "completions/max_length": 472.8666666666667, + "completions/max_terminated_length": 434.1333333333333, + "completions/mean_length": 168.57472915649413, + "completions/mean_terminated_length": 165.08282343546549, + "completions/min_length": 44.9, + "completions/min_terminated_length": 44.9, + "entropy": 0.12957718757291634, + "epoch": 0.02734211825466145, + "eval/gt_acc_batch": 0.7411111235618592, + "frac_reward_zero_std": 1.0, + "grad_norm": 57.847511291503906, + "kd/policy_loss": -20.79253362019857, + "kd/rkl_advantage_mean": 116.04290472666422, + "kd/rkl_advantage_p95": 146.22897491455078, + "kd/rkl_advantage_std": 40.95622863769531, + "kd/ssd_loss": 0.0, + "learning_rate": 9.72695856909581e-07, + "loss": -83.17014973958334, + "num_tokens": 22995271.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7411111156145732, + "rewards/gt_logging_reward/std": 0.4298926015694936, + "sampling/importance_sampling_ratio/max": 2.195249052842458, + "sampling/importance_sampling_ratio/mean": 0.9999056140581767, + "sampling/importance_sampling_ratio/min": 0.32628690674901006, + "sampling/sampling_logp_difference/max": 1.0097139288981756, + "sampling/sampling_logp_difference/mean": 0.004491190689926346, + "step": 720, + "step_time": 7.542232342184676, + "student/entropy": 0.12957718757291634 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0075000004221995676, + "completions/max_length": 452.76666666666665, + "completions/max_terminated_length": 416.5, + "completions/mean_length": 169.0988972981771, + "completions/mean_terminated_length": 166.50445353190105, + "completions/min_length": 37.13333333333333, + "completions/min_terminated_length": 37.13333333333333, + "entropy": 0.12666605692356825, + "epoch": 0.028481373181939013, + "eval/gt_acc_batch": 0.7700000127156575, + "frac_reward_zero_std": 1.0, + "grad_norm": 72.30036926269531, + "kd/policy_loss": -20.598707707722983, + "kd/rkl_advantage_mean": 115.86374944051107, + "kd/rkl_advantage_p95": 147.6635986328125, + "kd/rkl_advantage_std": 41.48799557685852, + "kd/ssd_loss": 0.0, + "learning_rate": 9.715566019823034e-07, + "loss": -82.39484049479167, + "num_tokens": 23905835.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7699999988079071, + "rewards/gt_logging_reward/std": 0.41423672437667847, + "sampling/importance_sampling_ratio/max": 2.1069735566775005, + "sampling/importance_sampling_ratio/mean": 0.9898354729016622, + "sampling/importance_sampling_ratio/min": 0.2828458055543403, + "sampling/sampling_logp_difference/max": 1.0820647021134695, + "sampling/sampling_logp_difference/mean": 0.004527548467740417, + "step": 750, + "step_time": 7.523234947999784, + "student/entropy": 0.12666605692356825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01277777859941125, + "completions/max_length": 466.8666666666667, + "completions/max_terminated_length": 427.5, + "completions/mean_length": 172.87417602539062, + "completions/mean_terminated_length": 168.58108469645182, + "completions/min_length": 47.233333333333334, + "completions/min_terminated_length": 47.233333333333334, + "entropy": 0.1262664672608177, + "epoch": 0.029620628109216573, + "eval/gt_acc_batch": 0.7497222403685252, + "frac_reward_zero_std": 1.0, + "grad_norm": 47.218143463134766, + "kd/policy_loss": -21.22592460314433, + "kd/rkl_advantage_mean": 118.60937798817953, + "kd/rkl_advantage_p95": 148.75506019592285, + "kd/rkl_advantage_std": 41.01243810653686, + "kd/ssd_loss": 0.0, + "learning_rate": 9.70417347055026e-07, + "loss": -84.9037109375, + "num_tokens": 24834126.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7497222244739532, + "rewards/gt_logging_reward/std": 0.4274419665336609, + "sampling/importance_sampling_ratio/max": 2.211591112613678, + "sampling/importance_sampling_ratio/mean": 0.9924570401509603, + "sampling/importance_sampling_ratio/min": 0.22638102266937493, + "sampling/sampling_logp_difference/max": 1.1578882575035094, + "sampling/sampling_logp_difference/mean": 0.004496504234460493, + "step": 780, + "step_time": 7.765783192989572, + "student/entropy": 0.1262664672608177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.012222222952793042, + "completions/max_length": 476.2, + "completions/max_terminated_length": 422.1333333333333, + "completions/mean_length": 168.73861999511718, + "completions/mean_terminated_length": 164.52700754801432, + "completions/min_length": 33.833333333333336, + "completions/min_terminated_length": 33.833333333333336, + "entropy": 0.12730178503940504, + "epoch": 0.030759883036494132, + "eval/gt_acc_batch": 0.7544444660345714, + "frac_reward_zero_std": 1.0, + "grad_norm": 59.346290588378906, + "kd/policy_loss": -21.628309615453084, + "kd/rkl_advantage_mean": 119.53369623819987, + "kd/rkl_advantage_p95": 149.88161760965983, + "kd/rkl_advantage_std": 41.545198917388916, + "kd/ssd_loss": 0.0, + "learning_rate": 9.692780921277484e-07, + "loss": -86.51324055989583, + "num_tokens": 25747009.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.754444440205892, + "rewards/gt_logging_reward/std": 0.4223755657672882, + "sampling/importance_sampling_ratio/max": 2.0820003191630048, + "sampling/importance_sampling_ratio/mean": 1.007675572236379, + "sampling/importance_sampling_ratio/min": 0.28925419176618256, + "sampling/sampling_logp_difference/max": 1.2577937086423239, + "sampling/sampling_logp_difference/mean": 0.0043857367321227985, + "step": 810, + "step_time": 7.701732468379972, + "student/entropy": 0.12730178503940504 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.012777778475234905, + "completions/max_length": 462.56666666666666, + "completions/max_terminated_length": 412.2, + "completions/mean_length": 169.83000946044922, + "completions/mean_terminated_length": 165.45010426839193, + "completions/min_length": 38.7, + "completions/min_terminated_length": 38.7, + "entropy": 0.12330981170137724, + "epoch": 0.031899137963771695, + "eval/gt_acc_batch": 0.7477778057257335, + "frac_reward_zero_std": 1.0, + "grad_norm": 55.44533920288086, + "kd/policy_loss": -21.80795742670695, + "kd/rkl_advantage_mean": 120.31087595621744, + "kd/rkl_advantage_p95": 150.9032969156901, + "kd/rkl_advantage_std": 41.18380085627238, + "kd/ssd_loss": 0.0, + "learning_rate": 9.68138837200471e-07, + "loss": -87.23182779947916, + "num_tokens": 26663789.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7477777798970541, + "rewards/gt_logging_reward/std": 0.42528433601061505, + "sampling/importance_sampling_ratio/max": 1.9807268977165222, + "sampling/importance_sampling_ratio/mean": 1.0023825625578562, + "sampling/importance_sampling_ratio/min": 0.3294269879659017, + "sampling/sampling_logp_difference/max": 0.9785956859588623, + "sampling/sampling_logp_difference/mean": 0.004173204442486167, + "step": 840, + "step_time": 7.6051830363886745, + "student/entropy": 0.12330981170137724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013888889675339062, + "completions/max_length": 446.93333333333334, + "completions/max_terminated_length": 416.23333333333335, + "completions/mean_length": 170.7238978068034, + "completions/mean_terminated_length": 165.94773559570314, + "completions/min_length": 41.86666666666667, + "completions/min_terminated_length": 41.86666666666667, + "entropy": 0.1252144243568182, + "epoch": 0.033038392891049255, + "eval/gt_acc_batch": 0.7252778053283692, + "frac_reward_zero_std": 1.0, + "grad_norm": 45.611873626708984, + "kd/policy_loss": -21.105814798672995, + "kd/rkl_advantage_mean": 117.26853211720784, + "kd/rkl_advantage_p95": 151.55719197591145, + "kd/rkl_advantage_std": 43.99506436983744, + "kd/ssd_loss": 0.0, + "learning_rate": 9.669995822731933e-07, + "loss": -84.42327473958333, + "num_tokens": 27585027.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7252777794996897, + "rewards/gt_logging_reward/std": 0.44327235519886016, + "sampling/importance_sampling_ratio/max": 1.9774612704912822, + "sampling/importance_sampling_ratio/mean": 1.0090025981267294, + "sampling/importance_sampling_ratio/min": 0.37221844544013344, + "sampling/sampling_logp_difference/max": 0.8352101902167003, + "sampling/sampling_logp_difference/mean": 0.0041235864860937, + "step": 870, + "step_time": 7.501285023948488, + "student/entropy": 0.1252144243568182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.011944445222616196, + "completions/max_length": 462.9, + "completions/max_terminated_length": 438.46666666666664, + "completions/mean_length": 168.44056447347006, + "completions/mean_terminated_length": 164.32713928222657, + "completions/min_length": 37.9, + "completions/min_terminated_length": 37.9, + "entropy": 0.12857392001897097, + "epoch": 0.034177647818326815, + "eval/gt_acc_batch": 0.7544444640477498, + "frac_reward_zero_std": 1.0, + "grad_norm": 41.890628814697266, + "kd/policy_loss": -21.371051247914632, + "kd/rkl_advantage_mean": 120.4287281036377, + "kd/rkl_advantage_p95": 152.3995449066162, + "kd/rkl_advantage_std": 42.62023245493571, + "kd/ssd_loss": 0.0, + "learning_rate": 9.658603273459157e-07, + "loss": -85.48421223958333, + "num_tokens": 28490381.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7544444461663564, + "rewards/gt_logging_reward/std": 0.42608392536640166, + "sampling/importance_sampling_ratio/max": 1.9570247371991476, + "sampling/importance_sampling_ratio/mean": 0.9958525617917379, + "sampling/importance_sampling_ratio/min": 0.3370188304533561, + "sampling/sampling_logp_difference/max": 0.9928297221660614, + "sampling/sampling_logp_difference/mean": 0.0041517920481661955, + "step": 900, + "step_time": 7.561329933834107, + "student/entropy": 0.12857392001897097 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.008888889383524657, + "completions/max_length": 426.6333333333333, + "completions/max_terminated_length": 410.1666666666667, + "completions/mean_length": 167.39778594970704, + "completions/mean_terminated_length": 164.29461415608725, + "completions/min_length": 33.1, + "completions/min_terminated_length": 33.1, + "entropy": 0.12374676863352457, + "epoch": 0.035316902745604374, + "eval/gt_acc_batch": 0.746666677792867, + "frac_reward_zero_std": 1.0, + "grad_norm": 43.38657760620117, + "kd/policy_loss": -21.123667621612547, + "kd/rkl_advantage_mean": 119.20782725016277, + "kd/rkl_advantage_p95": 153.17808354695637, + "kd/rkl_advantage_std": 44.19613865216573, + "kd/ssd_loss": 0.0, + "learning_rate": 9.647210724186382e-07, + "loss": -84.49466145833334, + "num_tokens": 29409469.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7466666678587596, + "rewards/gt_logging_reward/std": 0.4274430811405182, + "sampling/importance_sampling_ratio/max": 1.9708651224772136, + "sampling/importance_sampling_ratio/mean": 0.9960265318552654, + "sampling/importance_sampling_ratio/min": 0.37795839086174965, + "sampling/sampling_logp_difference/max": 0.8561015337705612, + "sampling/sampling_logp_difference/mean": 0.004052721631402771, + "step": 930, + "step_time": 7.474175309285056, + "student/entropy": 0.12374676863352457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.006388889160007238, + "completions/max_length": 450.1666666666667, + "completions/max_terminated_length": 412.03333333333336, + "completions/mean_length": 162.1975077311198, + "completions/mean_terminated_length": 159.90364735921224, + "completions/min_length": 30.666666666666668, + "completions/min_terminated_length": 30.666666666666668, + "entropy": 0.12297397665679455, + "epoch": 0.036456157672881934, + "eval/gt_acc_batch": 0.7502777854601542, + "frac_reward_zero_std": 1.0, + "grad_norm": 63.573402404785156, + "kd/policy_loss": -21.521896886825562, + "kd/rkl_advantage_mean": 121.29990634918212, + "kd/rkl_advantage_p95": 153.40335744222006, + "kd/rkl_advantage_std": 43.29854537645976, + "kd/ssd_loss": 0.0, + "learning_rate": 9.635818174913606e-07, + "loss": -86.08758951822917, + "num_tokens": 30298740.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7502777775128683, + "rewards/gt_logging_reward/std": 0.42469372153282164, + "sampling/importance_sampling_ratio/max": 1.8968185544013978, + "sampling/importance_sampling_ratio/mean": 0.9998959143956502, + "sampling/importance_sampling_ratio/min": 0.40594417651494347, + "sampling/sampling_logp_difference/max": 0.6703374753395717, + "sampling/sampling_logp_difference/mean": 0.003981719763639073, + "step": 960, + "step_time": 7.418338361033238, + "student/entropy": 0.12297397665679455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0055555558763444425, + "completions/max_length": 432.2, + "completions/max_terminated_length": 387.1666666666667, + "completions/mean_length": 155.6041737874349, + "completions/mean_terminated_length": 153.6367108662923, + "completions/min_length": 33.13333333333333, + "completions/min_terminated_length": 33.13333333333333, + "entropy": 0.12641709130257367, + "epoch": 0.03759541260015949, + "eval/gt_acc_batch": 0.7338889122009278, + "frac_reward_zero_std": 1.0, + "grad_norm": 52.22148513793945, + "kd/policy_loss": -21.26673003832499, + "kd/rkl_advantage_mean": 120.57805309295654, + "kd/rkl_advantage_p95": 153.8506098429362, + "kd/rkl_advantage_std": 44.182099930445354, + "kd/ssd_loss": 0.0, + "learning_rate": 9.62442562564083e-07, + "loss": -85.06692708333334, + "num_tokens": 31157259.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7338888843854269, + "rewards/gt_logging_reward/std": 0.43695598741372427, + "sampling/importance_sampling_ratio/max": 1.9629254778226217, + "sampling/importance_sampling_ratio/mean": 1.0013560553391774, + "sampling/importance_sampling_ratio/min": 0.4157633287832141, + "sampling/sampling_logp_difference/max": 0.7734236737092336, + "sampling/sampling_logp_difference/mean": 0.004119664942845702, + "step": 990, + "step_time": 7.187674804253038, + "student/entropy": 0.12641709130257367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01194444519157211, + "completions/max_length": 466.03333333333336, + "completions/max_terminated_length": 439.73333333333335, + "completions/mean_length": 163.7697301228841, + "completions/mean_terminated_length": 159.64504954020182, + "completions/min_length": 29.133333333333333, + "completions/min_terminated_length": 29.133333333333333, + "entropy": 0.12862137233217558, + "epoch": 0.03873466752743705, + "eval/gt_acc_batch": 0.7052778085072835, + "frac_reward_zero_std": 1.0, + "grad_norm": 60.77881622314453, + "kd/policy_loss": -22.441565815607706, + "kd/rkl_advantage_mean": 124.96527684529623, + "kd/rkl_advantage_p95": 154.30698254903157, + "kd/rkl_advantage_std": 41.56252967516581, + "kd/ssd_loss": 0.0, + "learning_rate": 9.613033076368056e-07, + "loss": -89.76626790364584, + "num_tokens": 32050494.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7052777747313181, + "rewards/gt_logging_reward/std": 0.453357274333636, + "sampling/importance_sampling_ratio/max": 2.0305368661880494, + "sampling/importance_sampling_ratio/mean": 0.9979638795057932, + "sampling/importance_sampling_ratio/min": 0.4208242133259773, + "sampling/sampling_logp_difference/max": 0.732830074429512, + "sampling/sampling_logp_difference/mean": 0.004147956380620599, + "step": 1020, + "step_time": 7.568541581222477, + "student/entropy": 0.12862137233217558 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01083333392937978, + "completions/max_length": 467.3, + "completions/max_terminated_length": 419.96666666666664, + "completions/mean_length": 160.44973169962566, + "completions/mean_terminated_length": 156.6212506612142, + "completions/min_length": 29.8, + "completions/min_terminated_length": 29.8, + "entropy": 0.126927571122845, + "epoch": 0.03987392245471462, + "eval/gt_acc_batch": 0.7377777953942617, + "frac_reward_zero_std": 1.0, + "grad_norm": 40.6567497253418, + "kd/policy_loss": -21.770963843663534, + "kd/rkl_advantage_mean": 123.14809837341309, + "kd/rkl_advantage_p95": 154.378088760376, + "kd/rkl_advantage_std": 42.96451317469279, + "kd/ssd_loss": 0.0, + "learning_rate": 9.60164052709528e-07, + "loss": -87.08387044270833, + "num_tokens": 32931857.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7377777814865112, + "rewards/gt_logging_reward/std": 0.4289328992366791, + "sampling/importance_sampling_ratio/max": 1.9423293908437094, + "sampling/importance_sampling_ratio/mean": 0.9986581206321716, + "sampling/importance_sampling_ratio/min": 0.4686017553011576, + "sampling/sampling_logp_difference/max": 0.6701851924260457, + "sampling/sampling_logp_difference/mean": 0.004097303414406876, + "step": 1050, + "step_time": 7.544307739330301, + "student/entropy": 0.126927571122845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009722222791363795, + "completions/max_length": 456.3666666666667, + "completions/max_terminated_length": 421.03333333333336, + "completions/mean_length": 157.42973047892252, + "completions/mean_terminated_length": 153.99574635823566, + "completions/min_length": 26.233333333333334, + "completions/min_terminated_length": 26.233333333333334, + "entropy": 0.13081179950386285, + "epoch": 0.04101317738199218, + "eval/gt_acc_batch": 0.7247222344080607, + "frac_reward_zero_std": 1.0, + "grad_norm": 26.785369873046875, + "kd/policy_loss": -22.52446052233378, + "kd/rkl_advantage_mean": 124.96490376790365, + "kd/rkl_advantage_p95": 154.6621561686198, + "kd/rkl_advantage_std": 41.76878355344137, + "kd/ssd_loss": 0.0, + "learning_rate": 9.590247977822503e-07, + "loss": -90.09784342447917, + "num_tokens": 33797820.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7247222244739533, + "rewards/gt_logging_reward/std": 0.44093614916006724, + "sampling/importance_sampling_ratio/max": 1.9961082061131796, + "sampling/importance_sampling_ratio/mean": 0.9959947208563487, + "sampling/importance_sampling_ratio/min": 0.365420317898194, + "sampling/sampling_logp_difference/max": 0.8000891377528508, + "sampling/sampling_logp_difference/mean": 0.004280803818255663, + "step": 1080, + "step_time": 7.514967222453561, + "student/entropy": 0.13081179950386285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.005555555907388528, + "completions/max_length": 445.23333333333335, + "completions/max_terminated_length": 409.23333333333335, + "completions/mean_length": 153.7825075785319, + "completions/mean_terminated_length": 151.80771967569987, + "completions/min_length": 21.433333333333334, + "completions/min_terminated_length": 21.433333333333334, + "entropy": 0.12850678730756043, + "epoch": 0.04215243230926974, + "eval/gt_acc_batch": 0.7619444648424785, + "frac_reward_zero_std": 1.0, + "grad_norm": 34.644508361816406, + "kd/policy_loss": -22.39489874839783, + "kd/rkl_advantage_mean": 123.69888038635254, + "kd/rkl_advantage_p95": 154.8365676879883, + "kd/rkl_advantage_std": 42.411386489868164, + "kd/ssd_loss": 0.0, + "learning_rate": 9.578855428549729e-07, + "loss": -89.57958984375, + "num_tokens": 34667101.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.761944442987442, + "rewards/gt_logging_reward/std": 0.4192190945148468, + "sampling/importance_sampling_ratio/max": 1.9835169474283854, + "sampling/importance_sampling_ratio/mean": 1.0069187084833782, + "sampling/importance_sampling_ratio/min": 0.4479299565156301, + "sampling/sampling_logp_difference/max": 0.5968128383159638, + "sampling/sampling_logp_difference/mean": 0.004219385120086372, + "step": 1110, + "step_time": 7.414739462081343, + "student/entropy": 0.12850678730756043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.007777778276552756, + "completions/max_length": 462.3666666666667, + "completions/max_terminated_length": 419.26666666666665, + "completions/mean_length": 151.58334197998047, + "completions/mean_terminated_length": 148.778054300944, + "completions/min_length": 23.6, + "completions/min_terminated_length": 23.6, + "entropy": 0.12400033076604207, + "epoch": 0.0432916872365473, + "eval/gt_acc_batch": 0.7316666980584462, + "frac_reward_zero_std": 1.0, + "grad_norm": 28.032325744628906, + "kd/policy_loss": -22.297235123316447, + "kd/rkl_advantage_mean": 125.01682504018147, + "kd/rkl_advantage_p95": 154.9810270945231, + "kd/rkl_advantage_std": 42.42968691190084, + "kd/ssd_loss": 0.0, + "learning_rate": 9.567462879276952e-07, + "loss": -89.18894856770834, + "num_tokens": 35513953.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7316666622956594, + "rewards/gt_logging_reward/std": 0.4386579364538193, + "sampling/importance_sampling_ratio/max": 1.8879525979359946, + "sampling/importance_sampling_ratio/mean": 0.9982172826925914, + "sampling/importance_sampling_ratio/min": 0.3990024487177531, + "sampling/sampling_logp_difference/max": 0.7326892594496409, + "sampling/sampling_logp_difference/mean": 0.00406228001229465, + "step": 1140, + "step_time": 7.3942477752221745, + "student/entropy": 0.12400033076604207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00611111136774222, + "completions/max_length": 402.43333333333334, + "completions/max_terminated_length": 364.6666666666667, + "completions/mean_length": 144.47084045410156, + "completions/mean_terminated_length": 142.2338788350423, + "completions/min_length": 18.833333333333332, + "completions/min_terminated_length": 18.833333333333332, + "entropy": 0.1262585292259852, + "epoch": 0.04443094216382486, + "eval/gt_acc_batch": 0.7172222415606181, + "frac_reward_zero_std": 1.0, + "grad_norm": 59.40279769897461, + "kd/policy_loss": -22.021988773345946, + "kd/rkl_advantage_mean": 123.18994077046712, + "kd/rkl_advantage_p95": 155.02737325032552, + "kd/rkl_advantage_std": 42.56041417121887, + "kd/ssd_loss": 0.0, + "learning_rate": 9.556070330004176e-07, + "loss": -88.08797200520833, + "num_tokens": 36342240.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7172222256660461, + "rewards/gt_logging_reward/std": 0.4444901059071223, + "sampling/importance_sampling_ratio/max": 1.8474903504053752, + "sampling/importance_sampling_ratio/mean": 0.9999083479245504, + "sampling/importance_sampling_ratio/min": 0.44167708307504655, + "sampling/sampling_logp_difference/max": 0.5179069578647614, + "sampling/sampling_logp_difference/mean": 0.004169638237605492, + "step": 1170, + "step_time": 7.019763613891943, + "student/entropy": 0.1262585292259852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00472222256163756, + "completions/max_length": 428.9, + "completions/max_terminated_length": 407.9, + "completions/mean_length": 147.99334106445312, + "completions/mean_terminated_length": 146.27495829264322, + "completions/min_length": 18.233333333333334, + "completions/min_terminated_length": 18.233333333333334, + "entropy": 0.12612100082139174, + "epoch": 0.04557019709110242, + "eval/gt_acc_batch": 0.6983333567778269, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.29359245300293, + "kd/policy_loss": -21.619527991612753, + "kd/rkl_advantage_mean": 123.80714155832926, + "kd/rkl_advantage_p95": 155.1161024729411, + "kd/rkl_advantage_std": 43.146903387705485, + "kd/ssd_loss": 0.0, + "learning_rate": 9.544677780731402e-07, + "loss": -86.47810872395833, + "num_tokens": 37181152.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.6983333359162013, + "rewards/gt_logging_reward/std": 0.446970342596372, + "sampling/importance_sampling_ratio/max": 1.8999436060587565, + "sampling/importance_sampling_ratio/mean": 0.9869282484054566, + "sampling/importance_sampling_ratio/min": 0.3732980584104856, + "sampling/sampling_logp_difference/max": 0.888214914004008, + "sampling/sampling_logp_difference/mean": 0.004282719517747561, + "step": 1200, + "step_time": 7.237560827455794, + "student/entropy": 0.12612100082139174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.006944444868713618, + "completions/max_length": 435.03333333333336, + "completions/max_terminated_length": 379.2, + "completions/mean_length": 140.43806381225585, + "completions/mean_terminated_length": 137.859011332194, + "completions/min_length": 12.4, + "completions/min_terminated_length": 12.4, + "entropy": 0.12816830407828092, + "epoch": 0.04670945201837998, + "eval/gt_acc_batch": 0.7036111305157343, + "frac_reward_zero_std": 1.0, + "grad_norm": 32.406402587890625, + "kd/policy_loss": -22.259174060821532, + "kd/rkl_advantage_mean": 124.84211616516113, + "kd/rkl_advantage_p95": 155.09651527404785, + "kd/rkl_advantage_std": 41.9529421488444, + "kd/ssd_loss": 0.0, + "learning_rate": 9.533285231458625e-07, + "loss": -89.03668619791667, + "num_tokens": 37990713.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7036111146211624, + "rewards/gt_logging_reward/std": 0.44838125109672544, + "sampling/importance_sampling_ratio/max": 1.8988737384478251, + "sampling/importance_sampling_ratio/mean": 1.002109036842982, + "sampling/importance_sampling_ratio/min": 0.41162156959374746, + "sampling/sampling_logp_difference/max": 0.5972098728020986, + "sampling/sampling_logp_difference/mean": 0.004379112451958159, + "step": 1230, + "step_time": 7.197091461163169, + "student/entropy": 0.12816830407828092 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.004444444707284371, + "completions/max_length": 421.93333333333334, + "completions/max_terminated_length": 385.1, + "completions/mean_length": 133.55361887613932, + "completions/mean_terminated_length": 131.89130884806315, + "completions/min_length": 9.766666666666667, + "completions/min_terminated_length": 9.766666666666667, + "entropy": 0.12574859242886305, + "epoch": 0.04784870694565754, + "eval/gt_acc_batch": 0.684444468220075, + "frac_reward_zero_std": 1.0, + "grad_norm": 29.622283935546875, + "kd/policy_loss": -21.55091278553009, + "kd/rkl_advantage_mean": 124.45806083679199, + "kd/rkl_advantage_p95": 155.19250729878743, + "kd/rkl_advantage_std": 42.492398230234784, + "kd/ssd_loss": 0.0, + "learning_rate": 9.52189268218585e-07, + "loss": -86.203662109375, + "num_tokens": 38776754.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.6844444463650385, + "rewards/gt_logging_reward/std": 0.4493180145819982, + "sampling/importance_sampling_ratio/max": 1.7854273835817973, + "sampling/importance_sampling_ratio/mean": 0.9948185543219249, + "sampling/importance_sampling_ratio/min": 0.383167905608813, + "sampling/sampling_logp_difference/max": 0.7440540989240011, + "sampling/sampling_logp_difference/mean": 0.0044004214073841775, + "step": 1260, + "step_time": 7.022708990533526, + "student/entropy": 0.12574859242886305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0025000001303851606, + "completions/max_length": 401.23333333333335, + "completions/max_terminated_length": 355.3, + "completions/mean_length": 113.04833857218425, + "completions/mean_terminated_length": 112.04689814249674, + "completions/min_length": 9.866666666666667, + "completions/min_terminated_length": 9.866666666666667, + "entropy": 0.12959002473702033, + "epoch": 0.0489879618729351, + "eval/gt_acc_batch": 0.6080555766820908, + "frac_reward_zero_std": 1.0, + "grad_norm": 34.159610748291016, + "kd/policy_loss": -22.276160168647767, + "kd/rkl_advantage_mean": 129.0088586807251, + "kd/rkl_advantage_p95": 155.2784428914388, + "kd/rkl_advantage_std": 39.560375197728476, + "kd/ssd_loss": 0.0, + "learning_rate": 9.510500132913074e-07, + "loss": -89.10464680989584, + "num_tokens": 39476912.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.6080555548270543, + "rewards/gt_logging_reward/std": 0.4823618580897649, + "sampling/importance_sampling_ratio/max": 1.980578323205312, + "sampling/importance_sampling_ratio/mean": 0.9988776127497355, + "sampling/importance_sampling_ratio/min": 0.35831866363684334, + "sampling/sampling_logp_difference/max": 0.8049303909142812, + "sampling/sampling_logp_difference/mean": 0.004902104241773486, + "step": 1290, + "step_time": 6.658571150782518, + "student/entropy": 0.12959002473702033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0025000001614292463, + "completions/max_length": 353.0, + "completions/max_terminated_length": 310.8, + "completions/mean_length": 102.8608383178711, + "completions/mean_terminated_length": 101.81637420654297, + "completions/min_length": 9.7, + "completions/min_terminated_length": 9.7, + "entropy": 0.12453161521504323, + "epoch": 0.05012721680021266, + "eval/gt_acc_batch": 0.5880555699268977, + "frac_reward_zero_std": 1.0, + "grad_norm": 35.496910095214844, + "kd/policy_loss": -20.45592356522878, + "kd/rkl_advantage_mean": 125.83674125671386, + "kd/rkl_advantage_p95": 155.22768567403156, + "kd/rkl_advantage_std": 43.18639583587647, + "kd/ssd_loss": 0.0, + "learning_rate": 9.499107583640299e-07, + "loss": -81.823681640625, + "num_tokens": 40150931.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.5880555560191473, + "rewards/gt_logging_reward/std": 0.4843309472004573, + "sampling/importance_sampling_ratio/max": 2.027687720457713, + "sampling/importance_sampling_ratio/mean": 0.9901000539461772, + "sampling/importance_sampling_ratio/min": 0.30910527855157854, + "sampling/sampling_logp_difference/max": 0.8950875441233317, + "sampling/sampling_logp_difference/mean": 0.005133619423334798, + "step": 1320, + "step_time": 6.351271408765267, + "student/entropy": 0.12453161521504323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0030555557770033676, + "completions/max_length": 317.6666666666667, + "completions/max_terminated_length": 266.53333333333336, + "completions/mean_length": 55.83333606719971, + "completions/mean_terminated_length": 54.402601432800296, + "completions/min_length": 9.933333333333334, + "completions/min_terminated_length": 9.933333333333334, + "entropy": 0.1322668192287286, + "epoch": 0.05126647172749022, + "eval/gt_acc_batch": 0.3511111189921697, + "frac_reward_zero_std": 1.0, + "grad_norm": 84.75296783447266, + "kd/policy_loss": -18.359119041760763, + "kd/rkl_advantage_mean": 134.59018433888752, + "kd/rkl_advantage_p95": 155.1076983133952, + "kd/rkl_advantage_std": 39.14837930401166, + "kd/ssd_loss": 0.0, + "learning_rate": 9.487715034367523e-07, + "loss": -73.436474609375, + "num_tokens": 40657075.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.351111110051473, + "rewards/gt_logging_reward/std": 0.4531140476465225, + "sampling/importance_sampling_ratio/max": 2.177434285481771, + "sampling/importance_sampling_ratio/mean": 0.9890071292718251, + "sampling/importance_sampling_ratio/min": 0.2487576810022195, + "sampling/sampling_logp_difference/max": 1.4696932236353557, + "sampling/sampling_logp_difference/mean": 0.009952307865023612, + "step": 1350, + "step_time": 5.820398722069028, + "student/entropy": 0.1322668192287286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0005555555845300357, + "completions/max_length": 179.0, + "completions/max_terminated_length": 169.2, + "completions/mean_length": 21.89611225128174, + "completions/mean_terminated_length": 21.622774187723795, + "completions/min_length": 10.633333333333333, + "completions/min_terminated_length": 10.633333333333333, + "entropy": 0.13636981267482043, + "epoch": 0.05240572665476778, + "eval/gt_acc_batch": 0.16472222656011581, + "frac_reward_zero_std": 1.0, + "grad_norm": 97.97065734863281, + "kd/policy_loss": -12.433423922459285, + "kd/rkl_advantage_mean": 104.48428835868836, + "kd/rkl_advantage_p95": 149.79541257222493, + "kd/rkl_advantage_std": 43.28249533971151, + "kd/ssd_loss": 0.0, + "learning_rate": 9.476322485094748e-07, + "loss": -49.73369140625, + "num_tokens": 41053549.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.16472222246229648, + "rewards/gt_logging_reward/std": 0.35881221344073616, + "sampling/importance_sampling_ratio/max": 2.002400585015615, + "sampling/importance_sampling_ratio/mean": 0.9951658129692078, + "sampling/importance_sampling_ratio/min": 0.28722271968921026, + "sampling/sampling_logp_difference/max": 1.3605454027652741, + "sampling/sampling_logp_difference/mean": 0.011073981768762071, + "step": 1380, + "step_time": 4.950961134559475, + "student/entropy": 0.13636981267482043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.2, + "completions/max_terminated_length": 14.2, + "completions/mean_length": 11.627778307596843, + "completions/mean_terminated_length": 11.627778307596843, + "completions/min_length": 9.933333333333334, + "completions/min_terminated_length": 9.933333333333334, + "entropy": 0.10957574074467023, + "epoch": 0.05354498158204534, + "eval/gt_acc_batch": 0.08583333582306901, + "frac_reward_zero_std": 1.0, + "grad_norm": 55.88983917236328, + "kd/policy_loss": -5.602865569790205, + "kd/rkl_advantage_mean": 33.509183692932126, + "kd/rkl_advantage_p95": 120.43653802871704, + "kd/rkl_advantage_std": 37.070027875900266, + "kd/ssd_loss": 0.0, + "learning_rate": 9.464929935821972e-07, + "loss": -22.411458333333332, + "num_tokens": 41393369.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.08583333507801096, + "rewards/gt_logging_reward/std": 0.24529974336425464, + "sampling/importance_sampling_ratio/max": 1.955947474638621, + "sampling/importance_sampling_ratio/mean": 0.9943344493707021, + "sampling/importance_sampling_ratio/min": 0.3672421539823214, + "sampling/sampling_logp_difference/max": 0.9672186275323232, + "sampling/sampling_logp_difference/mean": 0.014079808009167511, + "step": 1410, + "step_time": 4.276287125796079, + "student/entropy": 0.10957574074467023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.233333333333333, + "completions/max_terminated_length": 9.233333333333333, + "completions/mean_length": 5.3666669726371765, + "completions/mean_terminated_length": 5.3666669726371765, + "completions/min_length": 1.3333333333333333, + "completions/min_terminated_length": 1.3333333333333333, + "entropy": 0.08702319455333055, + "epoch": 0.0546842365093229, + "eval/gt_acc_batch": 0.013055556422720353, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.973514556884766, + "kd/policy_loss": -3.6227340541779993, + "kd/rkl_advantage_mean": 49.06711676021417, + "kd/rkl_advantage_p95": 95.02166035815026, + "kd/rkl_advantage_std": 37.028997089465456, + "kd/ssd_loss": 0.0, + "learning_rate": 9.453537386549196e-07, + "loss": -14.490937296549479, + "num_tokens": 41712745.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.013055555801838637, + "rewards/gt_logging_reward/std": 0.05551329875985781, + "sampling/importance_sampling_ratio/max": 1.4412250598271688, + "sampling/importance_sampling_ratio/mean": 0.996509724855423, + "sampling/importance_sampling_ratio/min": 0.5823668658733367, + "sampling/sampling_logp_difference/max": 0.6862912531554078, + "sampling/sampling_logp_difference/mean": 0.014996679531759583, + "step": 1440, + "step_time": 4.140091230588344, + "student/entropy": 0.08702319455333055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.4, + "completions/max_terminated_length": 4.4, + "completions/mean_length": 1.0472222566604614, + "completions/mean_terminated_length": 1.0472222566604614, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04601394119672477, + "epoch": 0.05582349143660046, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 32.81695556640625, + "kd/policy_loss": -1.528666998942693, + "kd/rkl_advantage_mean": 48.216686169306435, + "kd/rkl_advantage_p95": 76.64038521448771, + "kd/rkl_advantage_std": 36.07284233470758, + "kd/ssd_loss": 0.0, + "learning_rate": 9.442144837276421e-07, + "loss": -6.114668273925782, + "num_tokens": 42029939.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.277737597624461, + "sampling/importance_sampling_ratio/mean": 1.0008419056733449, + "sampling/importance_sampling_ratio/min": 0.9471359153588613, + "sampling/sampling_logp_difference/max": 0.24858185478951783, + "sampling/sampling_logp_difference/mean": 0.005462316810735501, + "step": 1470, + "step_time": 4.161182327704349, + "student/entropy": 0.04601394119672477 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 3.8666666666666667, + "completions/max_terminated_length": 3.8666666666666667, + "completions/mean_length": 1.0338889280954997, + "completions/mean_terminated_length": 1.0338889280954997, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03543563602336993, + "epoch": 0.05696274636387803, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.705846786499023, + "kd/policy_loss": -0.37983565827210747, + "kd/rkl_advantage_mean": 38.89777239163717, + "kd/rkl_advantage_p95": 65.88475759824117, + "kd/rkl_advantage_std": 33.38063674469789, + "kd/ssd_loss": 0.0, + "learning_rate": 9.430752288003644e-07, + "loss": -1.5193419138590494, + "num_tokens": 42341949.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.1530508041381835, + "sampling/importance_sampling_ratio/mean": 0.9999687810738881, + "sampling/importance_sampling_ratio/min": 0.9750884195168813, + "sampling/sampling_logp_difference/max": 0.1755512229981832, + "sampling/sampling_logp_difference/mean": 0.003686289676503899, + "step": 1500, + "step_time": 4.065437432817029, + "student/entropy": 0.03543563602336993 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.4, + "completions/max_terminated_length": 4.4, + "completions/mean_length": 1.0450000365575154, + "completions/mean_terminated_length": 1.0450000365575154, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05195645353135963, + "epoch": 0.058102001291155586, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 118.4114990234375, + "kd/policy_loss": -1.0572562714417775, + "kd/rkl_advantage_mean": 47.92974665959676, + "kd/rkl_advantage_p95": 77.0643219868342, + "kd/rkl_advantage_std": 37.92133183777332, + "kd/ssd_loss": 0.0, + "learning_rate": 9.41935973873087e-07, + "loss": -4.229027048746745, + "num_tokens": 42648743.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.004285195469856262, + "sampling/importance_sampling_ratio/max": 1.3494272232055664, + "sampling/importance_sampling_ratio/mean": 1.000771454970042, + "sampling/importance_sampling_ratio/min": 0.9798334817091624, + "sampling/sampling_logp_difference/max": 0.2623572147451341, + "sampling/sampling_logp_difference/mean": 0.005398342180221031, + "step": 1530, + "step_time": 4.115161807284069, + "student/entropy": 0.05195645353135963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.833333333333333, + "completions/max_terminated_length": 5.833333333333333, + "completions/mean_length": 1.0641667048136394, + "completions/mean_terminated_length": 1.0641667048136394, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.046838883132052915, + "epoch": 0.059241256218433146, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 32.481971740722656, + "kd/policy_loss": -3.2051993588606518, + "kd/rkl_advantage_mean": 60.08383905092875, + "kd/rkl_advantage_p95": 83.7607979853948, + "kd/rkl_advantage_std": 35.311397938927016, + "kd/ssd_loss": 0.0, + "learning_rate": 9.407967189458094e-07, + "loss": -12.820798746744792, + "num_tokens": 42962214.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.1681703289349874, + "sampling/importance_sampling_ratio/mean": 1.000404022137324, + "sampling/importance_sampling_ratio/min": 0.8518758366505305, + "sampling/sampling_logp_difference/max": 0.27914587447109324, + "sampling/sampling_logp_difference/mean": 0.0052871247384852415, + "step": 1560, + "step_time": 4.002106900567499, + "student/entropy": 0.046838883132052915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.9, + "completions/max_terminated_length": 5.9, + "completions/mean_length": 1.0675000309944154, + "completions/mean_terminated_length": 1.0675000309944154, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.026590700812327363, + "epoch": 0.060380511145710705, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.729696273803711, + "kd/policy_loss": -2.1338489294052123, + "kd/rkl_advantage_mean": 49.38353548049927, + "kd/rkl_advantage_p95": 66.90531134605408, + "kd/rkl_advantage_std": 27.97320786813895, + "kd/ssd_loss": 0.0, + "learning_rate": 9.396574640185319e-07, + "loss": -8.535396321614583, + "num_tokens": 43270057.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0215986847877503, + "sampling/importance_sampling_ratio/mean": 0.9995951990286509, + "sampling/importance_sampling_ratio/min": 0.7830423593521119, + "sampling/sampling_logp_difference/max": 0.2873059013703217, + "sampling/sampling_logp_difference/mean": 0.005380401401392495, + "step": 1590, + "step_time": 4.061504754909159, + "student/entropy": 0.026590700812327363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.433333333333334, + "completions/max_terminated_length": 7.433333333333334, + "completions/mean_length": 1.1108333786328635, + "completions/mean_terminated_length": 1.1108333786328635, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03998310922955473, + "epoch": 0.061519766072988265, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 293.3371887207031, + "kd/policy_loss": -5.448058244585991, + "kd/rkl_advantage_mean": 72.7528296470642, + "kd/rkl_advantage_p95": 95.40600101947784, + "kd/rkl_advantage_std": 34.655093183120094, + "kd/ssd_loss": 0.0, + "learning_rate": 9.385182090912542e-07, + "loss": -21.792232259114584, + "num_tokens": 43573304.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0561681747436524, + "sampling/importance_sampling_ratio/mean": 1.0002467215061188, + "sampling/importance_sampling_ratio/min": 0.7232438713312149, + "sampling/sampling_logp_difference/max": 0.38186579930285613, + "sampling/sampling_logp_difference/mean": 0.009225395577959716, + "step": 1620, + "step_time": 4.09601334615921, + "student/entropy": 0.03998310922955473 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.533333333333333, + "completions/max_terminated_length": 6.533333333333333, + "completions/mean_length": 1.0800000349680583, + "completions/mean_terminated_length": 1.0800000349680583, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03326301498649021, + "epoch": 0.06265902100026582, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.75295066833496, + "kd/policy_loss": -4.8275848468144735, + "kd/rkl_advantage_mean": 64.64130818049112, + "kd/rkl_advantage_p95": 84.01099563439688, + "kd/rkl_advantage_std": 30.90710004766782, + "kd/ssd_loss": 0.0, + "learning_rate": 9.373789541639767e-07, + "loss": -19.31033935546875, + "num_tokens": 43883984.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0412521839141846, + "sampling/importance_sampling_ratio/mean": 0.9988085707028707, + "sampling/importance_sampling_ratio/min": 0.7559363732735316, + "sampling/sampling_logp_difference/max": 0.33298884532414375, + "sampling/sampling_logp_difference/mean": 0.005236509155171613, + "step": 1650, + "step_time": 4.1943311886900725, + "student/entropy": 0.03326301498649021 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.533333333333333, + "completions/max_terminated_length": 6.533333333333333, + "completions/mean_length": 1.0761111537615458, + "completions/mean_terminated_length": 1.0761111537615458, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.060115114754686755, + "epoch": 0.06379827592754339, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 43.63157272338867, + "kd/policy_loss": -4.120061198870341, + "kd/rkl_advantage_mean": 66.43822042147319, + "kd/rkl_advantage_p95": 89.6418793519338, + "kd/rkl_advantage_std": 35.9871802786986, + "kd/ssd_loss": 0.0, + "learning_rate": 9.362396992366992e-07, + "loss": -16.480243937174478, + "num_tokens": 44198626.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.2444980462392172, + "sampling/importance_sampling_ratio/mean": 1.0001497348149617, + "sampling/importance_sampling_ratio/min": 0.802442126472791, + "sampling/sampling_logp_difference/max": 0.3405136616434902, + "sampling/sampling_logp_difference/mean": 0.006861465389374643, + "step": 1680, + "step_time": 4.198689443711191, + "student/entropy": 0.060115114754686755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.733333333333333, + "completions/max_terminated_length": 5.733333333333333, + "completions/mean_length": 1.06833336353302, + "completions/mean_terminated_length": 1.06833336353302, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03475175560452044, + "epoch": 0.06493753085482094, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.353628158569336, + "kd/policy_loss": -2.7278051733970643, + "kd/rkl_advantage_mean": 50.42962212562561, + "kd/rkl_advantage_p95": 67.18949348926544, + "kd/rkl_advantage_std": 27.17511296272278, + "kd/ssd_loss": 0.0, + "learning_rate": 9.351004443094216e-07, + "loss": -10.911221313476563, + "num_tokens": 44504968.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0439275741577148, + "sampling/importance_sampling_ratio/mean": 1.0001207451025644, + "sampling/importance_sampling_ratio/min": 0.8105579942464829, + "sampling/sampling_logp_difference/max": 0.25560670799265306, + "sampling/sampling_logp_difference/mean": 0.005096979152100781, + "step": 1710, + "step_time": 4.1930952242692, + "student/entropy": 0.03475175560452044 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.2, + "completions/max_terminated_length": 7.2, + "completions/mean_length": 1.0850000421206156, + "completions/mean_terminated_length": 1.0850000421206156, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03255132771252344, + "epoch": 0.06607678578209851, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 240.0553436279297, + "kd/policy_loss": -4.671743329366048, + "kd/rkl_advantage_mean": 69.16430465380351, + "kd/rkl_advantage_p95": 89.77601737976075, + "kd/rkl_advantage_std": 33.99944474101066, + "kd/ssd_loss": 0.0, + "learning_rate": 9.339611893821441e-07, + "loss": -18.68697306315104, + "num_tokens": 44807906.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0201011300086975, + "sampling/importance_sampling_ratio/mean": 0.9987452387809753, + "sampling/importance_sampling_ratio/min": 0.7403483315526197, + "sampling/sampling_logp_difference/max": 0.4256429750006646, + "sampling/sampling_logp_difference/mean": 0.00585847671997423, + "step": 1740, + "step_time": 4.112912669828317, + "student/entropy": 0.03255132771252344 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.533333333333333, + "completions/max_terminated_length": 5.533333333333333, + "completions/mean_length": 1.0650000373522441, + "completions/mean_terminated_length": 1.0650000373522441, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04605338067437212, + "epoch": 0.06721604070937606, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 138.6441650390625, + "kd/policy_loss": -3.2025222380956015, + "kd/rkl_advantage_mean": 56.35921411514282, + "kd/rkl_advantage_p95": 78.68835014502207, + "kd/rkl_advantage_std": 33.17130062580109, + "kd/ssd_loss": 0.0, + "learning_rate": 9.328219344548665e-07, + "loss": -12.810093180338542, + "num_tokens": 45112844.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.2349136432011922, + "sampling/importance_sampling_ratio/mean": 1.0005222459634144, + "sampling/importance_sampling_ratio/min": 0.8717582434415817, + "sampling/sampling_logp_difference/max": 0.23870732756331564, + "sampling/sampling_logp_difference/mean": 0.004411887226160616, + "step": 1770, + "step_time": 4.148813921849554, + "student/entropy": 0.04605338067437212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0561111450195313, + "completions/mean_terminated_length": 1.0561111450195313, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.039584930435133475, + "epoch": 0.06835529563665363, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 33.7457389831543, + "kd/policy_loss": -3.1223327944676083, + "kd/rkl_advantage_mean": 59.464844131469725, + "kd/rkl_advantage_p95": 84.0532090584437, + "kd/rkl_advantage_std": 35.402517760793366, + "kd/ssd_loss": 0.0, + "learning_rate": 9.316826795275889e-07, + "loss": -12.489329020182291, + "num_tokens": 45414878.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.157253587245941, + "sampling/importance_sampling_ratio/mean": 1.0003264685471853, + "sampling/importance_sampling_ratio/min": 0.8337229520082474, + "sampling/sampling_logp_difference/max": 0.29340559812262657, + "sampling/sampling_logp_difference/mean": 0.004723642904233809, + "step": 1800, + "step_time": 4.184716505475808, + "student/entropy": 0.039584930435133475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.3, + "completions/max_terminated_length": 6.3, + "completions/mean_length": 1.0708333730697632, + "completions/mean_terminated_length": 1.0708333730697632, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04884398275365432, + "epoch": 0.0694945505639312, + "eval/gt_acc_batch": 0.001388888992369175, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.689194679260254, + "kd/policy_loss": -4.158685886859894, + "kd/rkl_advantage_mean": 62.12129577000936, + "kd/rkl_advantage_p95": 83.72500275770822, + "kd/rkl_advantage_std": 32.88857742349307, + "kd/ssd_loss": 0.0, + "learning_rate": 9.305434246003113e-07, + "loss": -16.6347412109375, + "num_tokens": 45728317.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889302810033, + "rewards/gt_logging_reward/std": 0.011311792333920797, + "sampling/importance_sampling_ratio/max": 1.0834279855092366, + "sampling/importance_sampling_ratio/mean": 0.9997027734915416, + "sampling/importance_sampling_ratio/min": 0.812731654693683, + "sampling/sampling_logp_difference/max": 0.3539053102489561, + "sampling/sampling_logp_difference/mean": 0.0053763809555675834, + "step": 1830, + "step_time": 4.194861631033321, + "student/entropy": 0.04884398275365432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.833333333333333, + "completions/max_terminated_length": 6.833333333333333, + "completions/mean_length": 1.0991666992505391, + "completions/mean_terminated_length": 1.0991666992505391, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0567963570356369, + "epoch": 0.07063380549120875, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.778078079223633, + "kd/policy_loss": -4.334698084990183, + "kd/rkl_advantage_mean": 64.22236975034077, + "kd/rkl_advantage_p95": 84.24190524419149, + "kd/rkl_advantage_std": 31.879408516486485, + "kd/ssd_loss": 0.0, + "learning_rate": 9.294041696730338e-07, + "loss": -17.3387939453125, + "num_tokens": 46041962.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0088920434316, + "sampling/importance_sampling_ratio/mean": 1.0012113829453786, + "sampling/importance_sampling_ratio/min": 0.8023468732833863, + "sampling/sampling_logp_difference/max": 0.24025325540763637, + "sampling/sampling_logp_difference/mean": 0.006592016690410674, + "step": 1860, + "step_time": 4.185328084781456, + "student/entropy": 0.0567963570356369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0533333659172057, + "completions/mean_terminated_length": 1.0533333659172057, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.033914097119122744, + "epoch": 0.07177306041848631, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.833232879638672, + "kd/policy_loss": -3.2798266490300496, + "kd/rkl_advantage_mean": 60.50292828877767, + "kd/rkl_advantage_p95": 84.15548694928488, + "kd/rkl_advantage_std": 35.39747275412083, + "kd/ssd_loss": 0.0, + "learning_rate": 9.282649147457561e-07, + "loss": -13.11930643717448, + "num_tokens": 46353082.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.094314690430959, + "sampling/importance_sampling_ratio/mean": 0.9995418131351471, + "sampling/importance_sampling_ratio/min": 0.8633540292580922, + "sampling/sampling_logp_difference/max": 0.20827366192049038, + "sampling/sampling_logp_difference/mean": 0.0029466472391504795, + "step": 1890, + "step_time": 4.203293673290561, + "student/entropy": 0.033914097119122744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.866666666666666, + "completions/max_terminated_length": 5.866666666666666, + "completions/mean_length": 1.0605555812517802, + "completions/mean_terminated_length": 1.0605555812517802, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04057286737176279, + "epoch": 0.07291231534576387, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 32.06734848022461, + "kd/policy_loss": -2.5438063820203145, + "kd/rkl_advantage_mean": 53.568474515279135, + "kd/rkl_advantage_p95": 73.07263780434927, + "kd/rkl_advantage_std": 31.01466206908226, + "kd/ssd_loss": 0.0, + "learning_rate": 9.271256598184787e-07, + "loss": -10.17522481282552, + "num_tokens": 46662908.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0611720561981202, + "sampling/importance_sampling_ratio/mean": 0.9996084411938985, + "sampling/importance_sampling_ratio/min": 0.7980914317071438, + "sampling/sampling_logp_difference/max": 0.3013496356550604, + "sampling/sampling_logp_difference/mean": 0.0046443250748173645, + "step": 1920, + "step_time": 4.059435668909767, + "student/entropy": 0.04057286737176279 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.6, + "completions/max_terminated_length": 6.6, + "completions/mean_length": 1.0783333738644918, + "completions/mean_terminated_length": 1.0783333738644918, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03851471243736645, + "epoch": 0.07405157027304143, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 38.436100006103516, + "kd/policy_loss": -3.755563575029373, + "kd/rkl_advantage_mean": 60.392658376693724, + "kd/rkl_advantage_p95": 78.72627902825674, + "kd/rkl_advantage_std": 30.469891970356304, + "kd/ssd_loss": 0.0, + "learning_rate": 9.259864048912012e-07, + "loss": -15.022255452473958, + "num_tokens": 46976870.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0351077715555828, + "sampling/importance_sampling_ratio/mean": 0.9997971177101135, + "sampling/importance_sampling_ratio/min": 0.8298367281754812, + "sampling/sampling_logp_difference/max": 0.20881520357603828, + "sampling/sampling_logp_difference/mean": 0.0037910107367982466, + "step": 1950, + "step_time": 4.035031366138719, + "student/entropy": 0.03851471243736645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.0597222526868184, + "completions/mean_terminated_length": 1.0597222526868184, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05682929555575053, + "epoch": 0.07519082520031899, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.434255599975586, + "kd/policy_loss": -2.3531788110733034, + "kd/rkl_advantage_mean": 56.93980409304301, + "kd/rkl_advantage_p95": 78.88130548000336, + "kd/rkl_advantage_std": 34.61001346608003, + "kd/ssd_loss": 0.0, + "learning_rate": 9.248471499639236e-07, + "loss": -9.412715657552083, + "num_tokens": 47277037.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0420986612637837, + "sampling/importance_sampling_ratio/mean": 1.0005253632863362, + "sampling/importance_sampling_ratio/min": 0.8412576039632161, + "sampling/sampling_logp_difference/max": 0.20845380336977543, + "sampling/sampling_logp_difference/mean": 0.003996480791829527, + "step": 1980, + "step_time": 4.093912588692425, + "student/entropy": 0.05682929555575053 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.333333333333333, + "completions/max_terminated_length": 6.333333333333333, + "completions/mean_length": 1.087777821222941, + "completions/mean_terminated_length": 1.087777821222941, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.061078211696197586, + "epoch": 0.07633008012759655, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.912896156311035, + "kd/policy_loss": -3.7807715187470117, + "kd/rkl_advantage_mean": 64.43914254506429, + "kd/rkl_advantage_p95": 89.86793386936188, + "kd/rkl_advantage_std": 37.07635694245497, + "kd/ssd_loss": 0.0, + "learning_rate": 9.23707895036646e-07, + "loss": -15.123087565104166, + "num_tokens": 47588729.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.004285195469856262, + "sampling/importance_sampling_ratio/max": 1.0405884742736817, + "sampling/importance_sampling_ratio/mean": 1.0004379272460937, + "sampling/importance_sampling_ratio/min": 0.8150461296240489, + "sampling/sampling_logp_difference/max": 0.22947769993916153, + "sampling/sampling_logp_difference/mean": 0.005327409781360378, + "step": 2010, + "step_time": 4.168965895714549, + "student/entropy": 0.061078211696197586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.9, + "completions/max_terminated_length": 5.9, + "completions/mean_length": 1.0536111434300741, + "completions/mean_terminated_length": 1.0536111434300741, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.026458679310356576, + "epoch": 0.0774693350548741, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.344043731689453, + "kd/policy_loss": -1.7772613207499186, + "kd/rkl_advantage_mean": 52.082485389709475, + "kd/rkl_advantage_p95": 72.91302754878998, + "kd/rkl_advantage_std": 32.701343568166095, + "kd/ssd_loss": 0.0, + "learning_rate": 9.225686401093684e-07, + "loss": -7.109042358398438, + "num_tokens": 47893922.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.026350979010264, + "sampling/importance_sampling_ratio/mean": 0.9995163003603618, + "sampling/importance_sampling_ratio/min": 0.8465562303860982, + "sampling/sampling_logp_difference/max": 0.1878742894390598, + "sampling/sampling_logp_difference/mean": 0.002688930410658941, + "step": 2040, + "step_time": 4.103186282309859, + "student/entropy": 0.026458679310356576 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.533333333333333, + "completions/max_terminated_length": 6.533333333333333, + "completions/mean_length": 1.0611111481984457, + "completions/mean_terminated_length": 1.0611111481984457, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04637544754271706, + "epoch": 0.07860858998215167, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 58.90118408203125, + "kd/policy_loss": -3.121958148479462, + "kd/rkl_advantage_mean": 65.18310182889303, + "kd/rkl_advantage_p95": 89.76287989616394, + "kd/rkl_advantage_std": 38.54371716479461, + "kd/ssd_loss": 0.0, + "learning_rate": 9.214293851820909e-07, + "loss": -12.487832641601562, + "num_tokens": 48200702.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0231953303019206, + "sampling/importance_sampling_ratio/mean": 0.9997796595096589, + "sampling/importance_sampling_ratio/min": 0.8289715409278869, + "sampling/sampling_logp_difference/max": 0.21692185543167095, + "sampling/sampling_logp_difference/mean": 0.0035201654808285337, + "step": 2070, + "step_time": 4.114897169110676, + "student/entropy": 0.04637544754271706 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.0722222526868184, + "completions/mean_terminated_length": 1.0722222526868184, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.045105445695420104, + "epoch": 0.07974784490942924, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.81491470336914, + "kd/policy_loss": -3.727474554379781, + "kd/rkl_advantage_mean": 67.2232959429423, + "kd/rkl_advantage_p95": 89.91638284524282, + "kd/rkl_advantage_std": 36.93638544181983, + "kd/ssd_loss": 0.0, + "learning_rate": 9.202901302548133e-07, + "loss": -14.909898885091145, + "num_tokens": 48512426.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0186776757240295, + "sampling/importance_sampling_ratio/mean": 1.000210020939509, + "sampling/importance_sampling_ratio/min": 0.7929538349310558, + "sampling/sampling_logp_difference/max": 0.2463059455466767, + "sampling/sampling_logp_difference/mean": 0.005042372973791013, + "step": 2100, + "step_time": 4.113231714415209, + "student/entropy": 0.045105445695420104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.333333333333333, + "completions/max_terminated_length": 5.333333333333333, + "completions/mean_length": 1.0516667048136392, + "completions/mean_terminated_length": 1.0516667048136392, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03521789849425356, + "epoch": 0.08088709983670679, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 89.05669403076172, + "kd/policy_loss": -2.5842286835114163, + "kd/rkl_advantage_mean": 51.979772154490156, + "kd/rkl_advantage_p95": 78.24889663855235, + "kd/rkl_advantage_std": 33.81075336039066, + "kd/ssd_loss": 0.0, + "learning_rate": 9.191508753275358e-07, + "loss": -10.336912027994792, + "num_tokens": 48816820.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.004285195469856262, + "sampling/importance_sampling_ratio/max": 1.0162289937337239, + "sampling/importance_sampling_ratio/mean": 0.9990955054759979, + "sampling/importance_sampling_ratio/min": 0.7861177066961924, + "sampling/sampling_logp_difference/max": 0.27056448279569545, + "sampling/sampling_logp_difference/mean": 0.0051396895665675405, + "step": 2130, + "step_time": 4.108528186194599, + "student/entropy": 0.03521789849425356 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.966666666666667, + "completions/max_terminated_length": 4.966666666666667, + "completions/mean_length": 1.0491667111714682, + "completions/mean_terminated_length": 1.0491667111714682, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05590395629405975, + "epoch": 0.08202635476398436, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.495996475219727, + "kd/policy_loss": -2.4938898493846255, + "kd/rkl_advantage_mean": 50.62814292907715, + "kd/rkl_advantage_p95": 83.63764464855194, + "kd/rkl_advantage_std": 37.0577251603206, + "kd/ssd_loss": 0.0, + "learning_rate": 9.180116204002582e-07, + "loss": -9.975560506184896, + "num_tokens": 49119237.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0877207001050313, + "sampling/importance_sampling_ratio/mean": 0.9999677181243897, + "sampling/importance_sampling_ratio/min": 0.8023576065897942, + "sampling/sampling_logp_difference/max": 0.30753323320920267, + "sampling/sampling_logp_difference/mean": 0.006733718937418113, + "step": 2160, + "step_time": 4.067795047183366, + "student/entropy": 0.05590395629405975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0619444767634074, + "completions/mean_terminated_length": 1.0619444767634074, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.038123758121704064, + "epoch": 0.08316560969126191, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.051895141601562, + "kd/policy_loss": -2.4599002877871197, + "kd/rkl_advantage_mean": 60.74152522087097, + "kd/rkl_advantage_p95": 84.32739386558532, + "kd/rkl_advantage_std": 37.064951931933564, + "kd/ssd_loss": 0.0, + "learning_rate": 9.168723654729806e-07, + "loss": -9.839605712890625, + "num_tokens": 49434276.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.010503311951955, + "sampling/importance_sampling_ratio/mean": 1.0000270028909048, + "sampling/importance_sampling_ratio/min": 0.839183755715688, + "sampling/sampling_logp_difference/max": 0.18816410466097294, + "sampling/sampling_logp_difference/mean": 0.0032840979731796932, + "step": 2190, + "step_time": 4.153960167608845, + "student/entropy": 0.038123758121704064 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.733333333333333, + "completions/max_terminated_length": 6.733333333333333, + "completions/mean_length": 1.0755555868148803, + "completions/mean_terminated_length": 1.0755555868148803, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04276967269058029, + "epoch": 0.08430486461853948, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 102.89177703857422, + "kd/policy_loss": -4.194197007020315, + "kd/rkl_advantage_mean": 60.24644519488017, + "kd/rkl_advantage_p95": 78.6476724545161, + "kd/rkl_advantage_std": 29.43140749335289, + "kd/ssd_loss": 0.0, + "learning_rate": 9.15733110545703e-07, + "loss": -16.776793416341146, + "num_tokens": 49742980.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0518158713976542, + "sampling/importance_sampling_ratio/mean": 1.0005059699217478, + "sampling/importance_sampling_ratio/min": 0.7927460506558418, + "sampling/sampling_logp_difference/max": 0.28556623118929564, + "sampling/sampling_logp_difference/mean": 0.005457028540937851, + "step": 2220, + "step_time": 4.060534237841299, + "student/entropy": 0.04276967269058029 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.066666666666666, + "completions/max_terminated_length": 6.066666666666666, + "completions/mean_length": 1.0550000270207722, + "completions/mean_terminated_length": 1.0550000270207722, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.030133014575888715, + "epoch": 0.08544411954581703, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.201414108276367, + "kd/policy_loss": -2.0986600955327352, + "kd/rkl_advantage_mean": 53.60091072718303, + "kd/rkl_advantage_p95": 73.01645317077637, + "kd/rkl_advantage_std": 31.624758727351825, + "kd/ssd_loss": 0.0, + "learning_rate": 9.145938556184255e-07, + "loss": -8.394640604654947, + "num_tokens": 50050266.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0138740499814352, + "sampling/importance_sampling_ratio/mean": 1.000223696231842, + "sampling/importance_sampling_ratio/min": 0.8671441237131755, + "sampling/sampling_logp_difference/max": 0.1857563093615075, + "sampling/sampling_logp_difference/mean": 0.0032617695222143086, + "step": 2250, + "step_time": 4.109771019942127, + "student/entropy": 0.030133014575888715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.0561111450195313, + "completions/mean_terminated_length": 1.0561111450195313, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.038313950241232915, + "epoch": 0.0865833744730946, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.5810489654541, + "kd/policy_loss": -1.7555289030075074, + "kd/rkl_advantage_mean": 55.98250495592753, + "kd/rkl_advantage_p95": 78.45361872514088, + "kd/rkl_advantage_std": 35.199848400553066, + "kd/ssd_loss": 0.0, + "learning_rate": 9.134546006911479e-07, + "loss": -7.022115071614583, + "num_tokens": 50348820.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0034643093744913, + "sampling/importance_sampling_ratio/mean": 1.0000132183233896, + "sampling/importance_sampling_ratio/min": 0.8186938643455506, + "sampling/sampling_logp_difference/max": 0.21665505630274615, + "sampling/sampling_logp_difference/mean": 0.0036630856765744587, + "step": 2280, + "step_time": 4.12829187278403, + "student/entropy": 0.038313950241232915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.466666666666667, + "completions/max_terminated_length": 6.466666666666667, + "completions/mean_length": 1.071666705608368, + "completions/mean_terminated_length": 1.071666705608368, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06867681453004479, + "epoch": 0.08772262940037216, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 140.83326721191406, + "kd/policy_loss": -4.470782240231832, + "kd/rkl_advantage_mean": 66.92653495470682, + "kd/rkl_advantage_p95": 89.91493492126465, + "kd/rkl_advantage_std": 35.86885983149211, + "kd/ssd_loss": 0.0, + "learning_rate": 9.123153457638704e-07, + "loss": -17.883133951822916, + "num_tokens": 50648422.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.141329026222229, + "sampling/importance_sampling_ratio/mean": 1.0003249009450277, + "sampling/importance_sampling_ratio/min": 0.8090352018674215, + "sampling/sampling_logp_difference/max": 0.2876805840060115, + "sampling/sampling_logp_difference/mean": 0.00519254869238163, + "step": 2310, + "step_time": 4.091786881027899, + "student/entropy": 0.06867681453004479 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.666666666666667, + "completions/max_terminated_length": 5.666666666666667, + "completions/mean_length": 1.0527778108914694, + "completions/mean_terminated_length": 1.0527778108914694, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03813458997756243, + "epoch": 0.08886188432764972, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 42.34110641479492, + "kd/policy_loss": -1.0272576212882996, + "kd/rkl_advantage_mean": 47.47864708900452, + "kd/rkl_advantage_p95": 67.30112492243448, + "kd/rkl_advantage_std": 31.474597772955896, + "kd/ssd_loss": 0.0, + "learning_rate": 9.111760908365929e-07, + "loss": -4.10903065999349, + "num_tokens": 50970220.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0069849054018656, + "sampling/importance_sampling_ratio/mean": 0.9998738606770833, + "sampling/importance_sampling_ratio/min": 0.8526715596516927, + "sampling/sampling_logp_difference/max": 0.17134597116770844, + "sampling/sampling_logp_difference/mean": 0.0029096416139509527, + "step": 2340, + "step_time": 4.294450180799079, + "student/entropy": 0.03813458997756243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.233333333333333, + "completions/max_terminated_length": 7.233333333333333, + "completions/mean_length": 1.083055599530538, + "completions/mean_terminated_length": 1.083055599530538, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04528667274862528, + "epoch": 0.09000113925492728, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 147.77662658691406, + "kd/policy_loss": -4.858326439062754, + "kd/rkl_advantage_mean": 73.11695446968079, + "kd/rkl_advantage_p95": 95.74414069652558, + "kd/rkl_advantage_std": 37.342664709687234, + "kd/ssd_loss": 0.0, + "learning_rate": 9.100368359093152e-07, + "loss": -19.43330281575521, + "num_tokens": 51274687.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.091958471139272, + "sampling/importance_sampling_ratio/mean": 0.9993551870187124, + "sampling/importance_sampling_ratio/min": 0.7557989175120989, + "sampling/sampling_logp_difference/max": 0.3425366653439899, + "sampling/sampling_logp_difference/mean": 0.004933963838266209, + "step": 2370, + "step_time": 4.170574587478768, + "student/entropy": 0.04528667274862528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.8, + "completions/max_terminated_length": 5.8, + "completions/mean_length": 1.0766667048136394, + "completions/mean_terminated_length": 1.0766667048136394, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06761429036657016, + "epoch": 0.09114039418220483, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 26.786800384521484, + "kd/policy_loss": -5.034772175550461, + "kd/rkl_advantage_mean": 63.31107095082601, + "kd/rkl_advantage_p95": 84.3306886434555, + "kd/rkl_advantage_std": 32.0012726833423, + "kd/ssd_loss": 0.0, + "learning_rate": 9.088975809820377e-07, + "loss": -20.1390869140625, + "num_tokens": 51566899.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.2814095497131348, + "sampling/importance_sampling_ratio/mean": 0.9995538055896759, + "sampling/importance_sampling_ratio/min": 0.8602291832367579, + "sampling/sampling_logp_difference/max": 0.2947440802430113, + "sampling/sampling_logp_difference/mean": 0.006927007320336997, + "step": 2400, + "step_time": 4.042620687535964, + "student/entropy": 0.06761429036657016 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.0600000341733298, + "completions/mean_terminated_length": 1.0600000341733298, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.044919934837768474, + "epoch": 0.0922796491094824, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 56.156715393066406, + "kd/policy_loss": -2.5059320251146953, + "kd/rkl_advantage_mean": 57.77210653622945, + "kd/rkl_advantage_p95": 78.56314620971679, + "kd/rkl_advantage_std": 33.93302861253421, + "kd/ssd_loss": 0.0, + "learning_rate": 9.077583260547601e-07, + "loss": -10.023726399739584, + "num_tokens": 51870691.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.019199534257253, + "sampling/importance_sampling_ratio/mean": 1.0004260996977488, + "sampling/importance_sampling_ratio/min": 0.804764132698377, + "sampling/sampling_logp_difference/max": 0.21194053143262864, + "sampling/sampling_logp_difference/mean": 0.004698633526762326, + "step": 2430, + "step_time": 4.141736469072445, + "student/entropy": 0.044919934837768474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.333333333333333, + "completions/max_terminated_length": 6.333333333333333, + "completions/mean_length": 1.066111155351003, + "completions/mean_terminated_length": 1.066111155351003, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.043050730042159556, + "epoch": 0.09341890403675995, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.93932342529297, + "kd/policy_loss": -3.977302548289299, + "kd/rkl_advantage_mean": 66.94395251274109, + "kd/rkl_advantage_p95": 95.28510386149088, + "kd/rkl_advantage_std": 39.85913806855679, + "kd/ssd_loss": 0.0, + "learning_rate": 9.066190711274826e-07, + "loss": -15.909210205078125, + "num_tokens": 52179681.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0470749974250793, + "sampling/importance_sampling_ratio/mean": 0.9998476068178813, + "sampling/importance_sampling_ratio/min": 0.7907563040653864, + "sampling/sampling_logp_difference/max": 0.33520842222496866, + "sampling/sampling_logp_difference/mean": 0.0065920276140483715, + "step": 2460, + "step_time": 4.173806928315511, + "student/entropy": 0.043050730042159556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.5, + "completions/max_terminated_length": 5.5, + "completions/mean_length": 1.0519444823265076, + "completions/mean_terminated_length": 1.0519444823265076, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03495316559759279, + "epoch": 0.09455815896403752, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 186.80291748046875, + "kd/policy_loss": -1.7895550549030304, + "kd/rkl_advantage_mean": 50.83981498082479, + "kd/rkl_advantage_p95": 77.96105959415436, + "kd/rkl_advantage_std": 35.57692855894565, + "kd/ssd_loss": 0.0, + "learning_rate": 9.054798162002049e-07, + "loss": -7.158220926920573, + "num_tokens": 52481588.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0230369528134664, + "sampling/importance_sampling_ratio/mean": 0.9995209952195485, + "sampling/importance_sampling_ratio/min": 0.8061132520437241, + "sampling/sampling_logp_difference/max": 0.28642968422888465, + "sampling/sampling_logp_difference/mean": 0.004892695470092198, + "step": 2490, + "step_time": 4.087440578135041, + "student/entropy": 0.03495316559759279 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.066666666666666, + "completions/max_terminated_length": 4.066666666666666, + "completions/mean_length": 1.036666703224182, + "completions/mean_terminated_length": 1.036666703224182, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03810039463763436, + "epoch": 0.09569741389131509, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 109.08964538574219, + "kd/policy_loss": -0.5098310788472493, + "kd/rkl_advantage_mean": 35.21075520515442, + "kd/rkl_advantage_p95": 61.54588236014048, + "kd/rkl_advantage_std": 30.40402154525121, + "kd/ssd_loss": 0.0, + "learning_rate": 9.043405612729275e-07, + "loss": -2.0393238067626953, + "num_tokens": 52786768.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0242588957150778, + "sampling/importance_sampling_ratio/mean": 0.9999880333741505, + "sampling/importance_sampling_ratio/min": 0.8907447169224422, + "sampling/sampling_logp_difference/max": 0.1613485685084015, + "sampling/sampling_logp_difference/mean": 0.0032292181878195454, + "step": 2520, + "step_time": 4.136135422849717, + "student/entropy": 0.03810039463763436 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.8, + "completions/max_terminated_length": 6.8, + "completions/mean_length": 1.0961111505826315, + "completions/mean_terminated_length": 1.0961111505826315, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.08381150069956979, + "epoch": 0.09683666881859264, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 30.286733627319336, + "kd/policy_loss": -5.5734048128128055, + "kd/rkl_advantage_mean": 72.95182674725851, + "kd/rkl_advantage_p95": 95.62709427674612, + "kd/rkl_advantage_std": 36.38861772914728, + "kd/ssd_loss": 0.0, + "learning_rate": 9.0320130634565e-07, + "loss": -22.293611653645833, + "num_tokens": 53094258.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.1177126884460449, + "sampling/importance_sampling_ratio/mean": 0.9999829153219859, + "sampling/importance_sampling_ratio/min": 0.8324870030085246, + "sampling/sampling_logp_difference/max": 0.231487090823551, + "sampling/sampling_logp_difference/mean": 0.006044318476536622, + "step": 2550, + "step_time": 4.147685621092872, + "student/entropy": 0.08381150069956979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.2, + "completions/max_terminated_length": 6.2, + "completions/mean_length": 1.0783333778381348, + "completions/mean_terminated_length": 1.0783333778381348, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04213373543073733, + "epoch": 0.0979759237458702, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 31.196691513061523, + "kd/policy_loss": -3.848536596695582, + "kd/rkl_advantage_mean": 61.5759340763092, + "kd/rkl_advantage_p95": 84.05657119750977, + "kd/rkl_advantage_std": 34.16660202592611, + "kd/ssd_loss": 0.0, + "learning_rate": 9.020620514183723e-07, + "loss": -15.394143676757812, + "num_tokens": 53405916.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.009921165307363, + "sampling/importance_sampling_ratio/mean": 0.9989503582318624, + "sampling/importance_sampling_ratio/min": 0.7934285156428814, + "sampling/sampling_logp_difference/max": 0.24671905424135426, + "sampling/sampling_logp_difference/mean": 0.005181806635422012, + "step": 2580, + "step_time": 4.158535625040531, + "student/entropy": 0.04213373543073733 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.0930555939674378, + "completions/mean_terminated_length": 1.0930555939674378, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06372007528940836, + "epoch": 0.09911517867314776, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.52202796936035, + "kd/policy_loss": -4.536343892415364, + "kd/rkl_advantage_mean": 60.75692947705587, + "kd/rkl_advantage_p95": 79.30944491227469, + "kd/rkl_advantage_std": 29.760835042595865, + "kd/ssd_loss": 0.0, + "learning_rate": 9.009227964910948e-07, + "loss": -18.14537353515625, + "num_tokens": 53721987.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.036296824614207, + "sampling/importance_sampling_ratio/mean": 0.9995848695437114, + "sampling/importance_sampling_ratio/min": 0.7895654688278834, + "sampling/sampling_logp_difference/max": 0.2333498623377333, + "sampling/sampling_logp_difference/mean": 0.005982563958968967, + "step": 2610, + "step_time": 4.208038574263143, + "student/entropy": 0.06372007528940836 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.433333333333334, + "completions/max_terminated_length": 5.433333333333334, + "completions/mean_length": 1.0480555931727091, + "completions/mean_terminated_length": 1.0480555931727091, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03951588619189958, + "epoch": 0.10025443360042532, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.355764389038086, + "kd/policy_loss": -2.2435282786687214, + "kd/rkl_advantage_mean": 50.89631209373474, + "kd/rkl_advantage_p95": 72.84003126621246, + "kd/rkl_advantage_std": 32.34372457265854, + "kd/ssd_loss": 0.0, + "learning_rate": 8.997835415638172e-07, + "loss": -8.974111938476563, + "num_tokens": 54035744.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0693354725837707, + "sampling/importance_sampling_ratio/mean": 1.0000884731610615, + "sampling/importance_sampling_ratio/min": 0.866094579299291, + "sampling/sampling_logp_difference/max": 0.16268943728258212, + "sampling/sampling_logp_difference/mean": 0.002866283916713049, + "step": 2640, + "step_time": 4.17128864179443, + "student/entropy": 0.03951588619189958 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.6, + "completions/max_terminated_length": 6.6, + "completions/mean_length": 1.0783333698908488, + "completions/mean_terminated_length": 1.0783333698908488, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0727669512697806, + "epoch": 0.10139368852770288, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.94339370727539, + "kd/policy_loss": -4.208822069565455, + "kd/rkl_advantage_mean": 68.22330773671469, + "kd/rkl_advantage_p95": 94.80811738173166, + "kd/rkl_advantage_std": 38.86316857536634, + "kd/ssd_loss": 0.0, + "learning_rate": 8.986442866365396e-07, + "loss": -16.835289510091147, + "num_tokens": 54333690.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.1464869817097982, + "sampling/importance_sampling_ratio/mean": 0.9991563975811004, + "sampling/importance_sampling_ratio/min": 0.8080170104900996, + "sampling/sampling_logp_difference/max": 0.2932385559969892, + "sampling/sampling_logp_difference/mean": 0.0059056848792048795, + "step": 2670, + "step_time": 4.090161205859234, + "student/entropy": 0.0727669512697806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.066666666666666, + "completions/max_terminated_length": 6.066666666666666, + "completions/mean_length": 1.0533333579699198, + "completions/mean_terminated_length": 1.0533333579699198, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.041811740150054295, + "epoch": 0.10253294345498044, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 83.49173736572266, + "kd/policy_loss": -1.3720621546109517, + "kd/rkl_advantage_mean": 51.61696170171102, + "kd/rkl_advantage_p95": 72.66131890614828, + "kd/rkl_advantage_std": 33.11554853816827, + "kd/ssd_loss": 0.0, + "learning_rate": 8.975050317092621e-07, + "loss": -5.488248189290364, + "num_tokens": 54626618.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0357468088467916, + "sampling/importance_sampling_ratio/mean": 0.9998970369497935, + "sampling/importance_sampling_ratio/min": 0.8023755207657814, + "sampling/sampling_logp_difference/max": 0.2503571512798468, + "sampling/sampling_logp_difference/mean": 0.0038910939474590123, + "step": 2700, + "step_time": 4.130243246478495, + "student/entropy": 0.041811740150054295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.366666666666666, + "completions/max_terminated_length": 6.366666666666666, + "completions/mean_length": 1.0486111442248027, + "completions/mean_terminated_length": 1.0486111442248027, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03534796085829536, + "epoch": 0.10367219838225801, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 46.30009841918945, + "kd/policy_loss": -2.3747258226076764, + "kd/rkl_advantage_mean": 59.492915344238284, + "kd/rkl_advantage_p95": 84.24823524157206, + "kd/rkl_advantage_std": 37.16477139691512, + "kd/ssd_loss": 0.0, + "learning_rate": 8.963657767819846e-07, + "loss": -9.498903401692708, + "num_tokens": 54926585.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0152876853942872, + "sampling/importance_sampling_ratio/mean": 0.9995455245176951, + "sampling/importance_sampling_ratio/min": 0.8169654250144959, + "sampling/sampling_logp_difference/max": 0.21994005786255003, + "sampling/sampling_logp_difference/mean": 0.0035028174403123557, + "step": 2730, + "step_time": 4.140028425325484, + "student/entropy": 0.03534796085829536 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.966666666666667, + "completions/max_terminated_length": 4.966666666666667, + "completions/mean_length": 1.0419444719950357, + "completions/mean_terminated_length": 1.0419444719950357, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.044243935386960705, + "epoch": 0.10481145330953556, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 40.37729263305664, + "kd/policy_loss": -1.8846612393856048, + "kd/rkl_advantage_mean": 49.55980391502381, + "kd/rkl_advantage_p95": 72.75311003526052, + "kd/rkl_advantage_std": 33.59981652498245, + "kd/ssd_loss": 0.0, + "learning_rate": 8.952265218547069e-07, + "loss": -7.538646443684896, + "num_tokens": 55223632.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.1302520990371705, + "sampling/importance_sampling_ratio/mean": 1.0008516669273377, + "sampling/importance_sampling_ratio/min": 0.897984512646993, + "sampling/sampling_logp_difference/max": 0.19115600438478092, + "sampling/sampling_logp_difference/mean": 0.003169499172751481, + "step": 2760, + "step_time": 4.13929822082088, + "student/entropy": 0.044243935386960705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.2, + "completions/max_terminated_length": 6.2, + "completions/mean_length": 1.0744445006052652, + "completions/mean_terminated_length": 1.0744445006052652, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06655007602336506, + "epoch": 0.10595070823681313, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 151.00674438476562, + "kd/policy_loss": -4.825362094243368, + "kd/rkl_advantage_mean": 73.42848409016928, + "kd/rkl_advantage_p95": 106.54931084314983, + "kd/rkl_advantage_std": 44.86791509886583, + "kd/ssd_loss": 0.0, + "learning_rate": 8.940872669274294e-07, + "loss": -19.301448567708334, + "num_tokens": 55525164.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.273328991731008, + "sampling/importance_sampling_ratio/mean": 0.999584831794103, + "sampling/importance_sampling_ratio/min": 0.8784855465094249, + "sampling/sampling_logp_difference/max": 0.29578705499880015, + "sampling/sampling_logp_difference/mean": 0.006524123294123759, + "step": 2790, + "step_time": 4.140073056984693, + "student/entropy": 0.06655007602336506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.666666666666667, + "completions/max_terminated_length": 5.666666666666667, + "completions/mean_length": 1.0438889185587565, + "completions/mean_terminated_length": 1.0438889185587565, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05305725562696655, + "epoch": 0.10708996316409068, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 43.9654655456543, + "kd/policy_loss": -1.6626280983289083, + "kd/rkl_advantage_mean": 53.62756587664286, + "kd/rkl_advantage_p95": 78.3296126127243, + "kd/rkl_advantage_std": 36.022797714670496, + "kd/ssd_loss": 0.0, + "learning_rate": 8.929480120001519e-07, + "loss": -6.6505126953125, + "num_tokens": 55821282.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.1100236177444458, + "sampling/importance_sampling_ratio/mean": 0.9985944430033366, + "sampling/importance_sampling_ratio/min": 0.8711031973361969, + "sampling/sampling_logp_difference/max": 0.20520746835197012, + "sampling/sampling_logp_difference/mean": 0.004121210243708143, + "step": 2820, + "step_time": 4.1029675370043455, + "student/entropy": 0.05305725562696655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.333333333333333, + "completions/max_terminated_length": 5.333333333333333, + "completions/mean_length": 1.090000037352244, + "completions/mean_terminated_length": 1.090000037352244, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07541700651248297, + "epoch": 0.10822921809136825, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 35.080345153808594, + "kd/policy_loss": -5.1241696616013845, + "kd/rkl_advantage_mean": 62.756991910934445, + "kd/rkl_advantage_p95": 84.05696702798208, + "kd/rkl_advantage_std": 31.70487406651179, + "kd/ssd_loss": 0.0, + "learning_rate": 8.918087570728742e-07, + "loss": -20.4966796875, + "num_tokens": 56142582.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.366449518998464, + "sampling/importance_sampling_ratio/mean": 1.0002887427806855, + "sampling/importance_sampling_ratio/min": 0.891496816277504, + "sampling/sampling_logp_difference/max": 0.3007836673253526, + "sampling/sampling_logp_difference/mean": 0.010512387605073552, + "step": 2850, + "step_time": 4.2236779845513714, + "student/entropy": 0.07541700651248297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.933333333333334, + "completions/max_terminated_length": 6.933333333333334, + "completions/mean_length": 1.0722222606341043, + "completions/mean_terminated_length": 1.0722222606341043, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.046614422214527926, + "epoch": 0.1093684730186458, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 26.162769317626953, + "kd/policy_loss": -3.167976415157318, + "kd/rkl_advantage_mean": 63.57379126548767, + "kd/rkl_advantage_p95": 84.40685578187306, + "kd/rkl_advantage_std": 35.011605336268744, + "kd/ssd_loss": 0.0, + "learning_rate": 8.906695021455968e-07, + "loss": -12.671907552083333, + "num_tokens": 56439210.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.007588525613149, + "sampling/importance_sampling_ratio/mean": 1.0000244776407878, + "sampling/importance_sampling_ratio/min": 0.8388720691204071, + "sampling/sampling_logp_difference/max": 0.18723903240946432, + "sampling/sampling_logp_difference/mean": 0.003519769245758653, + "step": 2880, + "step_time": 4.1655522780260075, + "student/entropy": 0.046614422214527926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.333333333333333, + "completions/max_terminated_length": 5.333333333333333, + "completions/mean_length": 1.0550000309944152, + "completions/mean_terminated_length": 1.0550000309944152, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03727679681032896, + "epoch": 0.11050772794592337, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 30.980518341064453, + "kd/policy_loss": -2.2178859651088714, + "kd/rkl_advantage_mean": 47.34546488126119, + "kd/rkl_advantage_p95": 66.92921053568521, + "kd/rkl_advantage_std": 28.895999972025553, + "kd/ssd_loss": 0.0, + "learning_rate": 8.895302472183192e-07, + "loss": -8.871543375651042, + "num_tokens": 56747248.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0415160417556764, + "sampling/importance_sampling_ratio/mean": 1.0004794279734293, + "sampling/importance_sampling_ratio/min": 0.8911012570063274, + "sampling/sampling_logp_difference/max": 0.14108041857058803, + "sampling/sampling_logp_difference/mean": 0.002652639365987852, + "step": 2910, + "step_time": 4.219803686335217, + "student/entropy": 0.03727679681032896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.966666666666667, + "completions/max_terminated_length": 5.966666666666667, + "completions/mean_length": 1.051388919353485, + "completions/mean_terminated_length": 1.051388919353485, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03482968908113738, + "epoch": 0.11164698287320092, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.253999710083008, + "kd/policy_loss": -1.4534436146418253, + "kd/rkl_advantage_mean": 51.74137312571208, + "kd/rkl_advantage_p95": 72.97404580911001, + "kd/rkl_advantage_std": 33.30770127375921, + "kd/ssd_loss": 0.0, + "learning_rate": 8.883909922910417e-07, + "loss": -5.813775126139323, + "num_tokens": 57046305.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0024565855662029, + "sampling/importance_sampling_ratio/mean": 0.9999965409437815, + "sampling/importance_sampling_ratio/min": 0.8703187525272369, + "sampling/sampling_logp_difference/max": 0.14203679982262354, + "sampling/sampling_logp_difference/mean": 0.0024870786796479176, + "step": 2940, + "step_time": 4.1320700805517845, + "student/entropy": 0.03482968908113738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.0697222590446471, + "completions/mean_terminated_length": 1.0697222590446471, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04578910976027449, + "epoch": 0.11278623780047849, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.11826515197754, + "kd/policy_loss": -3.134141031901042, + "kd/rkl_advantage_mean": 59.53720466295878, + "kd/rkl_advantage_p95": 78.1470053434372, + "kd/rkl_advantage_std": 31.979416737457118, + "kd/ssd_loss": 0.0, + "learning_rate": 8.87251737363764e-07, + "loss": -12.536564127604167, + "num_tokens": 57352380.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0034715056419372, + "sampling/importance_sampling_ratio/mean": 1.0001446863015493, + "sampling/importance_sampling_ratio/min": 0.8612454771995545, + "sampling/sampling_logp_difference/max": 0.16902327996989092, + "sampling/sampling_logp_difference/mean": 0.0035490664672882606, + "step": 2970, + "step_time": 4.181870595171737, + "student/entropy": 0.04578910976027449 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.933333333333334, + "completions/max_terminated_length": 5.933333333333334, + "completions/mean_length": 1.051111137866974, + "completions/mean_terminated_length": 1.051111137866974, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05199594351773461, + "epoch": 0.11392549272775605, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 149.8849334716797, + "kd/policy_loss": -0.8466761310895284, + "kd/rkl_advantage_mean": 47.90011657079061, + "kd/rkl_advantage_p95": 67.50442850589752, + "kd/rkl_advantage_std": 31.638240724802017, + "kd/ssd_loss": 0.0, + "learning_rate": 8.861124824364865e-07, + "loss": -3.386705271402995, + "num_tokens": 57658332.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.007469375928243, + "sampling/importance_sampling_ratio/mean": 1.0010008792082468, + "sampling/importance_sampling_ratio/min": 0.8985013584295909, + "sampling/sampling_logp_difference/max": 0.12929404408981401, + "sampling/sampling_logp_difference/mean": 0.002986461411152656, + "step": 3000, + "step_time": 4.109210507932585, + "student/entropy": 0.05199594351773461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0616666992505392, + "completions/mean_terminated_length": 1.0616666992505392, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054057753458619115, + "epoch": 0.1150647476550336, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.091157913208008, + "kd/policy_loss": -2.200658309459686, + "kd/rkl_advantage_mean": 57.38484603563945, + "kd/rkl_advantage_p95": 78.50663572152456, + "kd/rkl_advantage_std": 34.33543284088373, + "kd/ssd_loss": 0.0, + "learning_rate": 8.849732275092089e-07, + "loss": -8.802632649739584, + "num_tokens": 57963258.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0127817471822103, + "sampling/importance_sampling_ratio/mean": 1.0003007392088572, + "sampling/importance_sampling_ratio/min": 0.8381237477064133, + "sampling/sampling_logp_difference/max": 0.18554229298606514, + "sampling/sampling_logp_difference/mean": 0.003575058067993571, + "step": 3030, + "step_time": 4.165522675289927, + "student/entropy": 0.054057753458619115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.833333333333333, + "completions/max_terminated_length": 5.833333333333333, + "completions/mean_length": 1.0552778124809266, + "completions/mean_terminated_length": 1.0552778124809266, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.059837929376711445, + "epoch": 0.11620400258231117, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 206.41026306152344, + "kd/policy_loss": -2.8723746011654536, + "kd/rkl_advantage_mean": 55.032719167073566, + "kd/rkl_advantage_p95": 78.51758708953858, + "kd/rkl_advantage_std": 33.43268987039725, + "kd/ssd_loss": 0.0, + "learning_rate": 8.838339725819313e-07, + "loss": -11.489499918619792, + "num_tokens": 58277313.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0813956300417582, + "sampling/importance_sampling_ratio/mean": 1.0009786407152812, + "sampling/importance_sampling_ratio/min": 0.8397085299094518, + "sampling/sampling_logp_difference/max": 0.2082543820142746, + "sampling/sampling_logp_difference/mean": 0.004460098990239203, + "step": 3060, + "step_time": 4.147819793433882, + "student/entropy": 0.059837929376711445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.533333333333333, + "completions/max_terminated_length": 6.533333333333333, + "completions/mean_length": 1.0644444783528646, + "completions/mean_terminated_length": 1.0644444783528646, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04002249560629328, + "epoch": 0.11734325750958872, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.956600189208984, + "kd/policy_loss": -2.1023795505364737, + "kd/rkl_advantage_mean": 59.805355564753214, + "kd/rkl_advantage_p95": 84.14499435424804, + "kd/rkl_advantage_std": 37.72568562030792, + "kd/ssd_loss": 0.0, + "learning_rate": 8.826947176546539e-07, + "loss": -8.409517415364583, + "num_tokens": 58589881.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0107484817504884, + "sampling/importance_sampling_ratio/mean": 1.000252797206243, + "sampling/importance_sampling_ratio/min": 0.8901128133138021, + "sampling/sampling_logp_difference/max": 0.1252083791575084, + "sampling/sampling_logp_difference/mean": 0.0023824212645801404, + "step": 3090, + "step_time": 4.1668072040386805, + "student/entropy": 0.04002249560629328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.066666666666666, + "completions/max_terminated_length": 6.066666666666666, + "completions/mean_length": 1.0616667032241822, + "completions/mean_terminated_length": 1.0616667032241822, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047468242794275285, + "epoch": 0.11848251243686629, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 31.99295997619629, + "kd/policy_loss": -3.1552925268809, + "kd/rkl_advantage_mean": 58.417032877604164, + "kd/rkl_advantage_p95": 78.91600170930226, + "kd/rkl_advantage_std": 32.59353920519352, + "kd/ssd_loss": 0.0, + "learning_rate": 8.815554627273763e-07, + "loss": -12.621171061197916, + "num_tokens": 58896551.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0119576454162598, + "sampling/importance_sampling_ratio/mean": 1.0003487507502238, + "sampling/importance_sampling_ratio/min": 0.8591884493827819, + "sampling/sampling_logp_difference/max": 0.17770697937036553, + "sampling/sampling_logp_difference/mean": 0.0033054005587473513, + "step": 3120, + "step_time": 4.141716413110649, + "student/entropy": 0.047468242794275285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.766666666666667, + "completions/max_terminated_length": 5.766666666666667, + "completions/mean_length": 1.0491666913032531, + "completions/mean_terminated_length": 1.0491666913032531, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.02708196029998362, + "epoch": 0.11962176736414384, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 133.33737182617188, + "kd/policy_loss": -1.4264363447825115, + "kd/rkl_advantage_mean": 47.95346268018087, + "kd/rkl_advantage_p95": 67.34714713891347, + "kd/rkl_advantage_std": 30.3378757784764, + "kd/ssd_loss": 0.0, + "learning_rate": 8.804162078000987e-07, + "loss": -5.705746968587239, + "num_tokens": 59193728.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0059436003367106, + "sampling/importance_sampling_ratio/mean": 0.9996446907520294, + "sampling/importance_sampling_ratio/min": 0.8772594253222148, + "sampling/sampling_logp_difference/max": 0.13428095440225055, + "sampling/sampling_logp_difference/mean": 0.001889530306410355, + "step": 3150, + "step_time": 4.112306037444311, + "student/entropy": 0.02708196029998362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.7, + "completions/max_terminated_length": 6.7, + "completions/mean_length": 1.073055589199066, + "completions/mean_terminated_length": 1.073055589199066, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04485866342050334, + "epoch": 0.12076102229142141, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 40.935420989990234, + "kd/policy_loss": -3.7227844913800556, + "kd/rkl_advantage_mean": 63.53188473383586, + "kd/rkl_advantage_p95": 84.52451297442119, + "kd/rkl_advantage_std": 33.965199733773865, + "kd/ssd_loss": 0.0, + "learning_rate": 8.792769528728211e-07, + "loss": -14.891139729817708, + "num_tokens": 59497063.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0546281496683756, + "sampling/importance_sampling_ratio/mean": 1.000337109963099, + "sampling/importance_sampling_ratio/min": 0.8649392515420914, + "sampling/sampling_logp_difference/max": 0.1743057153808574, + "sampling/sampling_logp_difference/mean": 0.003315406118053943, + "step": 3180, + "step_time": 4.097470645501744, + "student/entropy": 0.04485866342050334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.666666666666667, + "completions/max_terminated_length": 5.666666666666667, + "completions/mean_length": 1.0633333683013917, + "completions/mean_terminated_length": 1.0633333683013917, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03477068492211401, + "epoch": 0.12190027721869898, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 78.74775695800781, + "kd/policy_loss": -2.054309093952179, + "kd/rkl_advantage_mean": 49.23493660291036, + "kd/rkl_advantage_p95": 67.75186460812887, + "kd/rkl_advantage_std": 29.072515762845676, + "kd/ssd_loss": 0.0, + "learning_rate": 8.781376979455436e-07, + "loss": -8.217237854003907, + "num_tokens": 59797795.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0162218769391378, + "sampling/importance_sampling_ratio/mean": 0.9997844239075978, + "sampling/importance_sampling_ratio/min": 0.8892819344997406, + "sampling/sampling_logp_difference/max": 0.12174784420058131, + "sampling/sampling_logp_difference/mean": 0.0020689673624777545, + "step": 3210, + "step_time": 4.146696864662227, + "student/entropy": 0.03477068492211401 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.366666666666666, + "completions/max_terminated_length": 6.366666666666666, + "completions/mean_length": 1.0930555899937948, + "completions/mean_terminated_length": 1.0930555899937948, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06749952578296264, + "epoch": 0.12303953214597653, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 59.975738525390625, + "kd/policy_loss": -3.8232696652412415, + "kd/rkl_advantage_mean": 59.68953461647034, + "kd/rkl_advantage_p95": 79.00486737092336, + "kd/rkl_advantage_std": 31.200511420766514, + "kd/ssd_loss": 0.0, + "learning_rate": 8.769984430182659e-07, + "loss": -15.293080647786459, + "num_tokens": 60124162.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.012094411253929137, + "sampling/importance_sampling_ratio/max": 1.0617451230684916, + "sampling/importance_sampling_ratio/mean": 0.999901431798935, + "sampling/importance_sampling_ratio/min": 0.8475421984990438, + "sampling/sampling_logp_difference/max": 0.19896897273138164, + "sampling/sampling_logp_difference/mean": 0.005117664257219682, + "step": 3240, + "step_time": 4.2493691206638085, + "student/entropy": 0.06749952578296264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.4, + "completions/max_terminated_length": 5.4, + "completions/mean_length": 1.0477778275807699, + "completions/mean_terminated_length": 1.0477778275807699, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06408565460393827, + "epoch": 0.1241787870732541, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 78.26746368408203, + "kd/policy_loss": -3.4040550698836642, + "kd/rkl_advantage_mean": 61.123773368199664, + "kd/rkl_advantage_p95": 100.4057234843572, + "kd/rkl_advantage_std": 44.159208341439566, + "kd/ssd_loss": 0.0, + "learning_rate": 8.758591880909884e-07, + "loss": -13.616221110026041, + "num_tokens": 60443870.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0778249740600585, + "sampling/importance_sampling_ratio/mean": 1.000613526503245, + "sampling/importance_sampling_ratio/min": 0.8379980504512787, + "sampling/sampling_logp_difference/max": 0.28086521790052454, + "sampling/sampling_logp_difference/mean": 0.005669356801081449, + "step": 3270, + "step_time": 4.208668637438677, + "student/entropy": 0.06408565460393827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.466666666666667, + "completions/max_terminated_length": 6.466666666666667, + "completions/mean_length": 1.0538889209429423, + "completions/mean_terminated_length": 1.0538889209429423, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.042442206665873526, + "epoch": 0.12531804200053165, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 28.677833557128906, + "kd/policy_loss": -2.482319011290868, + "kd/rkl_advantage_mean": 60.78981693585714, + "kd/rkl_advantage_p95": 84.15100537935892, + "kd/rkl_advantage_std": 36.97386214733124, + "kd/ssd_loss": 0.0, + "learning_rate": 8.747199331637109e-07, + "loss": -9.929276529947916, + "num_tokens": 60752408.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0326363563537597, + "sampling/importance_sampling_ratio/mean": 1.0002028067906699, + "sampling/importance_sampling_ratio/min": 0.8814227521419525, + "sampling/sampling_logp_difference/max": 0.17304204287938774, + "sampling/sampling_logp_difference/mean": 0.00277156523661688, + "step": 3300, + "step_time": 4.178482199041173, + "student/entropy": 0.042442206665873526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.4, + "completions/max_terminated_length": 5.4, + "completions/mean_length": 1.0516666968663533, + "completions/mean_terminated_length": 1.0516666968663533, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04181092406312625, + "epoch": 0.12645729692780922, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 44.94594955444336, + "kd/policy_loss": -2.6541440506776173, + "kd/rkl_advantage_mean": 52.22512416839599, + "kd/rkl_advantage_p95": 72.88874377409617, + "kd/rkl_advantage_std": 31.597578911979994, + "kd/ssd_loss": 0.0, + "learning_rate": 8.735806782364334e-07, + "loss": -10.616579182942708, + "num_tokens": 61059546.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0491772651672364, + "sampling/importance_sampling_ratio/mean": 1.0001407821973165, + "sampling/importance_sampling_ratio/min": 0.9247350871562958, + "sampling/sampling_logp_difference/max": 0.11114908227076133, + "sampling/sampling_logp_difference/mean": 0.001972811290761456, + "step": 3330, + "step_time": 4.163815961568616, + "student/entropy": 0.04181092406312625 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.0866666992505392, + "completions/mean_terminated_length": 1.0866666992505392, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05198170725877086, + "epoch": 0.12759655185508678, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 30.399240493774414, + "kd/policy_loss": -4.847395245234171, + "kd/rkl_advantage_mean": 61.12290641466777, + "kd/rkl_advantage_p95": 79.0491380929947, + "kd/rkl_advantage_std": 29.137255251407623, + "kd/ssd_loss": 0.0, + "learning_rate": 8.724414233091558e-07, + "loss": -19.389581298828126, + "num_tokens": 61371370.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.1200530131657918, + "sampling/importance_sampling_ratio/mean": 1.0005930423736573, + "sampling/importance_sampling_ratio/min": 0.8691164453824362, + "sampling/sampling_logp_difference/max": 0.2007724400454511, + "sampling/sampling_logp_difference/mean": 0.003949004351549472, + "step": 3360, + "step_time": 4.132085267039171, + "student/entropy": 0.05198170725877086 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.0772222558657327, + "completions/mean_terminated_length": 1.0772222558657327, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04686778454730908, + "epoch": 0.12873580678236432, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.270023345947266, + "kd/policy_loss": -3.5853765825430552, + "kd/rkl_advantage_mean": 57.52322928110758, + "kd/rkl_advantage_p95": 78.84497768084208, + "kd/rkl_advantage_std": 31.276435739795367, + "kd/ssd_loss": 0.0, + "learning_rate": 8.713021683818782e-07, + "loss": -14.34150390625, + "num_tokens": 61675928.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0893090804417929, + "sampling/importance_sampling_ratio/mean": 1.0001863261063895, + "sampling/importance_sampling_ratio/min": 0.8610774546861648, + "sampling/sampling_logp_difference/max": 0.20079606506042183, + "sampling/sampling_logp_difference/mean": 0.0036875632513935367, + "step": 3390, + "step_time": 4.127006882855979, + "student/entropy": 0.04686778454730908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.533333333333333, + "completions/max_terminated_length": 6.533333333333333, + "completions/mean_length": 1.0572222550710042, + "completions/mean_terminated_length": 1.0572222550710042, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03728518749897679, + "epoch": 0.1298750617096419, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.537403106689453, + "kd/policy_loss": -2.1487496733665465, + "kd/rkl_advantage_mean": 60.44257357915242, + "kd/rkl_advantage_p95": 84.5754601240158, + "kd/rkl_advantage_std": 37.74144493192434, + "kd/ssd_loss": 0.0, + "learning_rate": 8.701629134546007e-07, + "loss": -8.594999186197917, + "num_tokens": 61990230.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.017028307914734, + "sampling/importance_sampling_ratio/mean": 0.9997822721799214, + "sampling/importance_sampling_ratio/min": 0.8560204495986302, + "sampling/sampling_logp_difference/max": 0.190058449562639, + "sampling/sampling_logp_difference/mean": 0.0026955573994200677, + "step": 3420, + "step_time": 4.144163273690113, + "student/entropy": 0.03728518749897679 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.0875000357627869, + "completions/mean_terminated_length": 1.0875000357627869, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06513580431540807, + "epoch": 0.13101431663691945, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.517030715942383, + "kd/policy_loss": -3.961833868424098, + "kd/rkl_advantage_mean": 59.11374966303507, + "kd/rkl_advantage_p95": 78.68705701033274, + "kd/rkl_advantage_std": 31.196438243985178, + "kd/ssd_loss": 0.0, + "learning_rate": 8.69023658527323e-07, + "loss": -15.847337849934895, + "num_tokens": 62304433.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.005748411019643, + "sampling/importance_sampling_ratio/mean": 1.0004543741544087, + "sampling/importance_sampling_ratio/min": 0.867455408970515, + "sampling/sampling_logp_difference/max": 0.16175521266025802, + "sampling/sampling_logp_difference/mean": 0.0038993490277789535, + "step": 3450, + "step_time": 4.185335185169242, + "student/entropy": 0.06513580431540807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.833333333333333, + "completions/max_terminated_length": 5.833333333333333, + "completions/mean_length": 1.063611137866974, + "completions/mean_terminated_length": 1.063611137866974, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03538600062020123, + "epoch": 0.13215357156419702, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 32.651737213134766, + "kd/policy_loss": -3.0726770440737408, + "kd/rkl_advantage_mean": 51.16794163386027, + "kd/rkl_advantage_p95": 67.63133350213369, + "kd/rkl_advantage_std": 26.84406727552414, + "kd/ssd_loss": 0.0, + "learning_rate": 8.678844036000456e-07, + "loss": -12.290709431966146, + "num_tokens": 62614646.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0018644730250041, + "sampling/importance_sampling_ratio/mean": 0.999717100461324, + "sampling/importance_sampling_ratio/min": 0.9043903271357219, + "sampling/sampling_logp_difference/max": 0.11165770773465435, + "sampling/sampling_logp_difference/mean": 0.002119341508174936, + "step": 3480, + "step_time": 4.214684705083103, + "student/entropy": 0.03538600062020123 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.8, + "completions/max_terminated_length": 5.8, + "completions/mean_length": 1.0563889185587565, + "completions/mean_terminated_length": 1.0563889185587565, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04078773783209423, + "epoch": 0.1332928264914746, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 47.0778694152832, + "kd/policy_loss": -1.359369707107544, + "kd/rkl_advantage_mean": 48.21788169542948, + "kd/rkl_advantage_p95": 67.50978811581929, + "kd/rkl_advantage_std": 30.77657937010129, + "kd/ssd_loss": 0.0, + "learning_rate": 8.66745148672768e-07, + "loss": -5.437480672200521, + "num_tokens": 62917089.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0021479288736979, + "sampling/importance_sampling_ratio/mean": 0.9999369045098623, + "sampling/importance_sampling_ratio/min": 0.8975057800610861, + "sampling/sampling_logp_difference/max": 0.11005625138059258, + "sampling/sampling_logp_difference/mean": 0.002103095308605892, + "step": 3510, + "step_time": 4.140517999864339, + "student/entropy": 0.04078773783209423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.133333333333334, + "completions/max_terminated_length": 7.133333333333334, + "completions/mean_length": 1.0861111521720885, + "completions/mean_terminated_length": 1.0861111521720885, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05045888371144732, + "epoch": 0.13443208141875213, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 120.09521484375, + "kd/policy_loss": -4.266466069221496, + "kd/rkl_advantage_mean": 71.96704074541728, + "kd/rkl_advantage_p95": 95.81730884710947, + "kd/rkl_advantage_std": 39.13794966240724, + "kd/ssd_loss": 0.0, + "learning_rate": 8.656058937454904e-07, + "loss": -17.065867106119793, + "num_tokens": 63214375.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0322655042012532, + "sampling/importance_sampling_ratio/mean": 0.9999991118907928, + "sampling/importance_sampling_ratio/min": 0.8631907065709432, + "sampling/sampling_logp_difference/max": 0.16064168407271306, + "sampling/sampling_logp_difference/mean": 0.002943911124020815, + "step": 3540, + "step_time": 4.16020160378733, + "student/entropy": 0.05045888371144732 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.766666666666667, + "completions/max_terminated_length": 5.766666666666667, + "completions/mean_length": 1.0897222598393759, + "completions/mean_terminated_length": 1.0897222598393759, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06466784020885825, + "epoch": 0.1355713363460297, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 26.633604049682617, + "kd/policy_loss": -5.1693868160247805, + "kd/rkl_advantage_mean": 63.28453696568807, + "kd/rkl_advantage_p95": 84.14893639882406, + "kd/rkl_advantage_std": 31.997429185112317, + "kd/ssd_loss": 0.0, + "learning_rate": 8.644666388182128e-07, + "loss": -20.677547200520834, + "num_tokens": 63520458.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.1858500997225443, + "sampling/importance_sampling_ratio/mean": 1.0001812795797984, + "sampling/importance_sampling_ratio/min": 0.9108155528704326, + "sampling/sampling_logp_difference/max": 0.1919705554532508, + "sampling/sampling_logp_difference/mean": 0.0052815671796755245, + "step": 3570, + "step_time": 4.188974627045294, + "student/entropy": 0.06466784020885825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.933333333333334, + "completions/max_terminated_length": 5.933333333333334, + "completions/mean_length": 1.0708333611488343, + "completions/mean_terminated_length": 1.0708333611488343, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05645159520208835, + "epoch": 0.13671059127330726, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.065685272216797, + "kd/policy_loss": -3.4451458851496377, + "kd/rkl_advantage_mean": 55.02031812667847, + "kd/rkl_advantage_p95": 72.94013657569886, + "kd/rkl_advantage_std": 29.148235226670902, + "kd/ssd_loss": 0.0, + "learning_rate": 8.633273838909353e-07, + "loss": -13.780584716796875, + "num_tokens": 63823697.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0113474527994792, + "sampling/importance_sampling_ratio/mean": 0.9999754031499227, + "sampling/importance_sampling_ratio/min": 0.8819573134183883, + "sampling/sampling_logp_difference/max": 0.12830794411711394, + "sampling/sampling_logp_difference/mean": 0.0027900552260689436, + "step": 3600, + "step_time": 4.262304086486498, + "student/entropy": 0.05645159520208835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.094444477558136, + "completions/mean_terminated_length": 1.094444477558136, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06265546365951498, + "epoch": 0.13784984620058482, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 71.43383026123047, + "kd/policy_loss": -5.9865244428316755, + "kd/rkl_advantage_mean": 58.590081230799356, + "kd/rkl_advantage_p95": 73.48987828890482, + "kd/rkl_advantage_std": 23.377579310536383, + "kd/ssd_loss": 0.0, + "learning_rate": 8.621881289636577e-07, + "loss": -23.946099853515626, + "num_tokens": 64133237.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0396722952524822, + "sampling/importance_sampling_ratio/mean": 1.0000039180119833, + "sampling/importance_sampling_ratio/min": 0.821500489115715, + "sampling/sampling_logp_difference/max": 0.21727616012406845, + "sampling/sampling_logp_difference/mean": 0.004719811464504649, + "step": 3630, + "step_time": 4.269654545510033, + "student/entropy": 0.06265546365951498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.0850000341733297, + "completions/mean_terminated_length": 1.0850000341733297, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05168181729192535, + "epoch": 0.1389891011278624, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 28.366722106933594, + "kd/policy_loss": -3.504608178138733, + "kd/rkl_advantage_mean": 55.34942388534546, + "kd/rkl_advantage_p95": 73.04598968029022, + "kd/rkl_advantage_std": 28.50583829085032, + "kd/ssd_loss": 0.0, + "learning_rate": 8.610488740363801e-07, + "loss": -14.018428548177083, + "num_tokens": 64441007.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0039796074231466, + "sampling/importance_sampling_ratio/mean": 1.00000572403272, + "sampling/importance_sampling_ratio/min": 0.887867138783137, + "sampling/sampling_logp_difference/max": 0.12488138450620075, + "sampling/sampling_logp_difference/mean": 0.0031428526427286366, + "step": 3660, + "step_time": 4.191450330486987, + "student/entropy": 0.05168181729192535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.071666697661082, + "completions/mean_terminated_length": 1.071666697661082, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04435062473639846, + "epoch": 0.14012835605513993, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.58696937561035, + "kd/policy_loss": -2.7453287402788797, + "kd/rkl_advantage_mean": 58.39904157320658, + "kd/rkl_advantage_p95": 78.80791946252187, + "kd/rkl_advantage_std": 33.36124441027641, + "kd/ssd_loss": 0.0, + "learning_rate": 8.599096191091027e-07, + "loss": -10.981315104166667, + "num_tokens": 64764513.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0022608955701193, + "sampling/importance_sampling_ratio/mean": 0.9996794601281483, + "sampling/importance_sampling_ratio/min": 0.8758826484282811, + "sampling/sampling_logp_difference/max": 0.1387240670931836, + "sampling/sampling_logp_difference/mean": 0.0025395380003222575, + "step": 3690, + "step_time": 4.375792697366948, + "student/entropy": 0.04435062473639846 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.073055589199066, + "completions/mean_terminated_length": 1.073055589199066, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05831226278096437, + "epoch": 0.1412676109824175, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 51.572654724121094, + "kd/policy_loss": -3.388437799612681, + "kd/rkl_advantage_mean": 59.553694025675455, + "kd/rkl_advantage_p95": 79.3654286623001, + "kd/rkl_advantage_std": 31.969955585400264, + "kd/ssd_loss": 0.0, + "learning_rate": 8.58770364181825e-07, + "loss": -13.553750610351562, + "num_tokens": 65066288.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0401896635691326, + "sampling/importance_sampling_ratio/mean": 1.0005487004915874, + "sampling/importance_sampling_ratio/min": 0.8854670782883962, + "sampling/sampling_logp_difference/max": 0.14142247314254444, + "sampling/sampling_logp_difference/mean": 0.0030918986614172656, + "step": 3720, + "step_time": 4.271169429179281, + "student/entropy": 0.05831226278096437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.366666666666666, + "completions/max_terminated_length": 6.366666666666666, + "completions/mean_length": 1.0652778188387553, + "completions/mean_terminated_length": 1.0652778188387553, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03837587920327981, + "epoch": 0.14240686590969506, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.77637767791748, + "kd/policy_loss": -3.4431335111459096, + "kd/rkl_advantage_mean": 61.40710010528564, + "kd/rkl_advantage_p95": 84.13451763788859, + "kd/rkl_advantage_std": 34.88268505583207, + "kd/ssd_loss": 0.0, + "learning_rate": 8.576311092545475e-07, + "loss": -13.7725341796875, + "num_tokens": 65381515.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0115901350975036, + "sampling/importance_sampling_ratio/mean": 0.9997717559337616, + "sampling/importance_sampling_ratio/min": 0.869927587111791, + "sampling/sampling_logp_difference/max": 0.16270497959728042, + "sampling/sampling_logp_difference/mean": 0.0025900799199007453, + "step": 3750, + "step_time": 4.271514302561991, + "student/entropy": 0.03837587920327981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.3, + "completions/max_terminated_length": 5.3, + "completions/mean_length": 1.0469444751739503, + "completions/mean_terminated_length": 1.0469444751739503, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04006268756153683, + "epoch": 0.14354612083697263, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.022048950195312, + "kd/policy_loss": -1.442552963892619, + "kd/rkl_advantage_mean": 47.20741589864095, + "kd/rkl_advantage_p95": 67.56095213890076, + "kd/rkl_advantage_std": 31.103122828404107, + "kd/ssd_loss": 0.0, + "learning_rate": 8.564918543272699e-07, + "loss": -5.77021230061849, + "num_tokens": 65686868.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0308351198832193, + "sampling/importance_sampling_ratio/mean": 1.000394743680954, + "sampling/importance_sampling_ratio/min": 0.9116247812906901, + "sampling/sampling_logp_difference/max": 0.12558449015486986, + "sampling/sampling_logp_difference/mean": 0.0020821956160943953, + "step": 3780, + "step_time": 4.2530092443067895, + "student/entropy": 0.04006268756153683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.068333375453949, + "completions/mean_terminated_length": 1.068333375453949, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03858019014199575, + "epoch": 0.14468537576425017, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 63.92665481567383, + "kd/policy_loss": -3.8922125498453775, + "kd/rkl_advantage_mean": 60.37084050178528, + "kd/rkl_advantage_p95": 79.05935802459717, + "kd/rkl_advantage_std": 31.23350424071153, + "kd/ssd_loss": 0.0, + "learning_rate": 8.553525993999924e-07, + "loss": -15.568853759765625, + "num_tokens": 65991706.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0218241333961486, + "sampling/importance_sampling_ratio/mean": 1.0001153886318206, + "sampling/importance_sampling_ratio/min": 0.9042196313540141, + "sampling/sampling_logp_difference/max": 0.1175721972482279, + "sampling/sampling_logp_difference/mean": 0.0020515378477284686, + "step": 3810, + "step_time": 4.239337676751893, + "student/entropy": 0.03858019014199575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.0616667032241822, + "completions/mean_terminated_length": 1.0616667032241822, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04194723339751363, + "epoch": 0.14582463069152773, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.760354995727539, + "kd/policy_loss": -3.09697052637736, + "kd/rkl_advantage_mean": 58.08124424616496, + "kd/rkl_advantage_p95": 78.68997384707133, + "kd/rkl_advantage_std": 33.01872139275074, + "kd/ssd_loss": 0.0, + "learning_rate": 8.542133444727147e-07, + "loss": -12.387881469726562, + "num_tokens": 66292024.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.048789115746816, + "sampling/importance_sampling_ratio/mean": 1.0002536634604136, + "sampling/importance_sampling_ratio/min": 0.8874501864115397, + "sampling/sampling_logp_difference/max": 0.14658463795979818, + "sampling/sampling_logp_difference/mean": 0.002273749086695413, + "step": 3840, + "step_time": 4.240605150411526, + "student/entropy": 0.04194723339751363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.6, + "completions/max_terminated_length": 6.6, + "completions/mean_length": 1.0800000270207724, + "completions/mean_terminated_length": 1.0800000270207724, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04740299282905956, + "epoch": 0.1469638856188053, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.145082473754883, + "kd/policy_loss": -4.029718546072642, + "kd/rkl_advantage_mean": 60.41941680908203, + "kd/rkl_advantage_p95": 78.71274508635203, + "kd/rkl_advantage_std": 30.50907044907411, + "kd/ssd_loss": 0.0, + "learning_rate": 8.530740895454373e-07, + "loss": -16.1188720703125, + "num_tokens": 66598760.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0042389114697774, + "sampling/importance_sampling_ratio/mean": 0.9999324560165406, + "sampling/importance_sampling_ratio/min": 0.8812749783198038, + "sampling/sampling_logp_difference/max": 0.1375282435833166, + "sampling/sampling_logp_difference/mean": 0.002641735745904346, + "step": 3870, + "step_time": 4.21197934326871, + "student/entropy": 0.04740299282905956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.366666666666666, + "completions/max_terminated_length": 5.366666666666666, + "completions/mean_length": 1.0475000262260437, + "completions/mean_terminated_length": 1.0475000262260437, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0344240308739245, + "epoch": 0.14810314054608287, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.06243896484375, + "kd/policy_loss": -1.4383405407269796, + "kd/rkl_advantage_mean": 44.54930561383565, + "kd/rkl_advantage_p95": 62.13581740061442, + "kd/rkl_advantage_std": 27.831623910864195, + "kd/ssd_loss": 0.0, + "learning_rate": 8.519348346181598e-07, + "loss": -5.753364054361979, + "num_tokens": 66913963.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.02587331533432, + "sampling/importance_sampling_ratio/mean": 1.0000804245471955, + "sampling/importance_sampling_ratio/min": 0.912716148296992, + "sampling/sampling_logp_difference/max": 0.1197691870154813, + "sampling/sampling_logp_difference/mean": 0.0019399047324744365, + "step": 3900, + "step_time": 4.233552995553085, + "student/entropy": 0.0344240308739245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.8, + "completions/max_terminated_length": 6.8, + "completions/mean_length": 1.086666707197825, + "completions/mean_terminated_length": 1.086666707197825, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04270436420726279, + "epoch": 0.14924239547336043, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.402223587036133, + "kd/policy_loss": -3.6710850447416306, + "kd/rkl_advantage_mean": 65.97019750277201, + "kd/rkl_advantage_p95": 89.94604889551799, + "kd/rkl_advantage_std": 37.10175101757049, + "kd/ssd_loss": 0.0, + "learning_rate": 8.507955796908821e-07, + "loss": -14.684342447916666, + "num_tokens": 67218923.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0256142775217691, + "sampling/importance_sampling_ratio/mean": 0.9995502571264903, + "sampling/importance_sampling_ratio/min": 0.8554514706134796, + "sampling/sampling_logp_difference/max": 0.1764883011734734, + "sampling/sampling_logp_difference/mean": 0.0029823603729406995, + "step": 3930, + "step_time": 4.210734031310616, + "student/entropy": 0.04270436420726279 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.533333333333333, + "completions/max_terminated_length": 5.533333333333333, + "completions/mean_length": 1.0472222487131755, + "completions/mean_terminated_length": 1.0472222487131755, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03766873837448657, + "epoch": 0.15038165040063797, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 54.65504455566406, + "kd/policy_loss": -1.1021687944730123, + "kd/rkl_advantage_mean": 47.3337818145752, + "kd/rkl_advantage_p95": 67.01273588339488, + "kd/rkl_advantage_std": 31.49653957684835, + "kd/ssd_loss": 0.0, + "learning_rate": 8.496563247636046e-07, + "loss": -4.408675130208334, + "num_tokens": 67525837.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0021660447120666, + "sampling/importance_sampling_ratio/mean": 1.0001431584358216, + "sampling/importance_sampling_ratio/min": 0.9185866117477417, + "sampling/sampling_logp_difference/max": 0.0965160713220636, + "sampling/sampling_logp_difference/mean": 0.0018505131418351084, + "step": 3960, + "step_time": 4.208980739108908, + "student/entropy": 0.03766873837448657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.9, + "completions/max_terminated_length": 5.9, + "completions/mean_length": 1.0586111466089885, + "completions/mean_terminated_length": 1.0586111466089885, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03722472946780423, + "epoch": 0.15152090532791554, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.322548866271973, + "kd/policy_loss": -2.1549731413523356, + "kd/rkl_advantage_mean": 53.337298742930095, + "kd/rkl_advantage_p95": 72.97698357899984, + "kd/rkl_advantage_std": 32.08050064841906, + "kd/ssd_loss": 0.0, + "learning_rate": 8.48517069836327e-07, + "loss": -8.61989237467448, + "num_tokens": 67835400.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0141765793164572, + "sampling/importance_sampling_ratio/mean": 0.9999612073103586, + "sampling/importance_sampling_ratio/min": 0.9009591440359751, + "sampling/sampling_logp_difference/max": 0.12653874438256024, + "sampling/sampling_logp_difference/mean": 0.0021433778398204594, + "step": 3990, + "step_time": 4.128793804207817, + "student/entropy": 0.03722472946780423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.933333333333334, + "completions/max_terminated_length": 6.933333333333334, + "completions/mean_length": 1.0783333738644918, + "completions/mean_terminated_length": 1.0783333738644918, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.044718733864525954, + "epoch": 0.1526601602551931, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.412708282470703, + "kd/policy_loss": -4.105169588327408, + "kd/rkl_advantage_mean": 67.92651743888855, + "kd/rkl_advantage_p95": 90.21731792291006, + "kd/rkl_advantage_std": 36.22906546493371, + "kd/ssd_loss": 0.0, + "learning_rate": 8.473778149090494e-07, + "loss": -16.4206787109375, + "num_tokens": 68150130.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.009218486150106, + "sampling/importance_sampling_ratio/mean": 0.9999205907185872, + "sampling/importance_sampling_ratio/min": 0.8872950832049052, + "sampling/sampling_logp_difference/max": 0.13408643953346958, + "sampling/sampling_logp_difference/mean": 0.0025289466080721467, + "step": 4020, + "step_time": 4.222070406559699, + "student/entropy": 0.044718733864525954 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.866666666666666, + "completions/max_terminated_length": 5.866666666666666, + "completions/mean_length": 1.054722253481547, + "completions/mean_terminated_length": 1.054722253481547, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03930464955046773, + "epoch": 0.15379941518247067, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 36.37430953979492, + "kd/policy_loss": -2.5404698967933657, + "kd/rkl_advantage_mean": 56.80791978836059, + "kd/rkl_advantage_p95": 78.59901404380798, + "kd/rkl_advantage_std": 34.53766448199749, + "kd/ssd_loss": 0.0, + "learning_rate": 8.462385599817718e-07, + "loss": -10.161881510416666, + "num_tokens": 68455959.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0429029623667398, + "sampling/importance_sampling_ratio/mean": 1.0002999703089397, + "sampling/importance_sampling_ratio/min": 0.9090805272261302, + "sampling/sampling_logp_difference/max": 0.1231542742267872, + "sampling/sampling_logp_difference/mean": 0.002057891013100743, + "step": 4050, + "step_time": 4.167301254269357, + "student/entropy": 0.03930464955046773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.066666666666666, + "completions/max_terminated_length": 6.066666666666666, + "completions/mean_length": 1.0544444799423218, + "completions/mean_terminated_length": 1.0544444799423218, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04482313689465324, + "epoch": 0.1549386701097482, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 30.145029067993164, + "kd/policy_loss": -2.464826633532842, + "kd/rkl_advantage_mean": 59.44432943662007, + "kd/rkl_advantage_p95": 84.0257559299469, + "kd/rkl_advantage_std": 37.85945578714212, + "kd/ssd_loss": 0.0, + "learning_rate": 8.450993050544944e-07, + "loss": -9.859304809570313, + "num_tokens": 68762299.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0924307187398274, + "sampling/importance_sampling_ratio/mean": 0.9998502155145009, + "sampling/importance_sampling_ratio/min": 0.8936741153399149, + "sampling/sampling_logp_difference/max": 0.16288381811852257, + "sampling/sampling_logp_difference/mean": 0.0026058405191482356, + "step": 4080, + "step_time": 4.263421259773895, + "student/entropy": 0.04482313689465324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.4, + "completions/max_terminated_length": 4.4, + "completions/mean_length": 1.0472222526868185, + "completions/mean_terminated_length": 1.0472222526868185, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05271867661116024, + "epoch": 0.15607792503702578, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 166.4865264892578, + "kd/policy_loss": -2.511627689997355, + "kd/rkl_advantage_mean": 44.053472709655765, + "kd/rkl_advantage_p95": 61.92368309497833, + "kd/rkl_advantage_std": 26.50797679523627, + "kd/ssd_loss": 0.0, + "learning_rate": 8.439600501272167e-07, + "loss": -10.046512858072917, + "num_tokens": 69073941.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.2119442343711853, + "sampling/importance_sampling_ratio/mean": 0.9998844166596731, + "sampling/importance_sampling_ratio/min": 0.9319230993588765, + "sampling/sampling_logp_difference/max": 0.20790336295031012, + "sampling/sampling_logp_difference/mean": 0.004631267109652981, + "step": 4110, + "step_time": 4.26686076989087, + "student/entropy": 0.05271867661116024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.466666666666667, + "completions/max_terminated_length": 6.466666666666667, + "completions/mean_length": 1.0966667095820108, + "completions/mean_terminated_length": 1.0966667095820108, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06033913504021863, + "epoch": 0.15721717996430334, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 39.5905647277832, + "kd/policy_loss": -4.479566740989685, + "kd/rkl_advantage_mean": 65.6089739004771, + "kd/rkl_advantage_p95": 89.82014635403951, + "kd/rkl_advantage_std": 36.427592399716374, + "kd/ssd_loss": 0.0, + "learning_rate": 8.428207951999392e-07, + "loss": -17.918269856770834, + "num_tokens": 69381929.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.319312071800232, + "sampling/importance_sampling_ratio/mean": 1.0003177205721536, + "sampling/importance_sampling_ratio/min": 0.8613956371943156, + "sampling/sampling_logp_difference/max": 0.2694835359773909, + "sampling/sampling_logp_difference/mean": 0.007705740564657996, + "step": 4140, + "step_time": 4.363996554984866, + "student/entropy": 0.06033913504021863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.0925000349680583, + "completions/mean_terminated_length": 1.0925000349680583, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06868479403977593, + "epoch": 0.1583564348915809, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.992881774902344, + "kd/policy_loss": -4.598989387353261, + "kd/rkl_advantage_mean": 61.11900267601013, + "kd/rkl_advantage_p95": 79.12056840260824, + "kd/rkl_advantage_std": 29.14685599307219, + "kd/ssd_loss": 0.0, + "learning_rate": 8.416815402726617e-07, + "loss": -18.39595743815104, + "num_tokens": 69679430.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0042195359865824, + "sampling/importance_sampling_ratio/mean": 0.9999874194463094, + "sampling/importance_sampling_ratio/min": 0.8454819798469544, + "sampling/sampling_logp_difference/max": 0.1618191297631711, + "sampling/sampling_logp_difference/mean": 0.004055866387595112, + "step": 4170, + "step_time": 4.250253244698979, + "student/entropy": 0.06868479403977593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.533333333333333, + "completions/max_terminated_length": 6.533333333333333, + "completions/mean_length": 1.068333371480306, + "completions/mean_terminated_length": 1.068333371480306, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04516474778453509, + "epoch": 0.15949568981885848, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 39.763797760009766, + "kd/policy_loss": -2.9129594922065736, + "kd/rkl_advantage_mean": 61.93193850517273, + "kd/rkl_advantage_p95": 84.47694292068482, + "kd/rkl_advantage_std": 36.13700413207213, + "kd/ssd_loss": 0.0, + "learning_rate": 8.40542285345384e-07, + "loss": -11.65183817545573, + "num_tokens": 69984868.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0173646330833435, + "sampling/importance_sampling_ratio/mean": 0.9996343334515889, + "sampling/importance_sampling_ratio/min": 0.8528128703435262, + "sampling/sampling_logp_difference/max": 0.15541336964815855, + "sampling/sampling_logp_difference/mean": 0.002898450028927376, + "step": 4200, + "step_time": 4.244604017472981, + "student/entropy": 0.04516474778453509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.333333333333333, + "completions/max_terminated_length": 5.333333333333333, + "completions/mean_length": 1.045555587609609, + "completions/mean_terminated_length": 1.045555587609609, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04242228933920463, + "epoch": 0.16063494474613602, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 38.722843170166016, + "kd/policy_loss": -1.142421027024587, + "kd/rkl_advantage_mean": 46.867196289698285, + "kd/rkl_advantage_p95": 67.48605751196543, + "kd/rkl_advantage_std": 31.754771768053374, + "kd/ssd_loss": 0.0, + "learning_rate": 8.394030304181065e-07, + "loss": -4.569685363769532, + "num_tokens": 70287672.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0028721253077189, + "sampling/importance_sampling_ratio/mean": 0.9999469359715779, + "sampling/importance_sampling_ratio/min": 0.882086702187856, + "sampling/sampling_logp_difference/max": 0.12544378048429886, + "sampling/sampling_logp_difference/mean": 0.0021609498440132787, + "step": 4230, + "step_time": 4.213402226415928, + "student/entropy": 0.04242228933920463 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.6, + "completions/max_terminated_length": 6.6, + "completions/mean_length": 1.0633333722750347, + "completions/mean_terminated_length": 1.0633333722750347, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049630766517172256, + "epoch": 0.16177419967341358, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.705608367919922, + "kd/policy_loss": -3.2694451848665875, + "kd/rkl_advantage_mean": 66.46660044987996, + "kd/rkl_advantage_p95": 90.47164662679036, + "kd/rkl_advantage_std": 39.04560463229815, + "kd/ssd_loss": 0.0, + "learning_rate": 8.38263775490829e-07, + "loss": -13.077783203125, + "num_tokens": 70597356.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0028408249219258, + "sampling/importance_sampling_ratio/mean": 0.9996677358945211, + "sampling/importance_sampling_ratio/min": 0.853747546672821, + "sampling/sampling_logp_difference/max": 0.16337087848223747, + "sampling/sampling_logp_difference/mean": 0.0029883028687133144, + "step": 4260, + "step_time": 4.202648316890312, + "student/entropy": 0.049630766517172256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.0766667127609253, + "completions/mean_terminated_length": 1.0766667127609253, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.057226090754071875, + "epoch": 0.16291345460069115, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 100.68158721923828, + "kd/policy_loss": -4.377227578560511, + "kd/rkl_advantage_mean": 59.85595653851827, + "kd/rkl_advantage_p95": 78.46408686637878, + "kd/rkl_advantage_std": 29.730793017148972, + "kd/ssd_loss": 0.0, + "learning_rate": 8.371245205635515e-07, + "loss": -17.5089111328125, + "num_tokens": 70896208.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0504416545232138, + "sampling/importance_sampling_ratio/mean": 1.000307591756185, + "sampling/importance_sampling_ratio/min": 0.8504507929086685, + "sampling/sampling_logp_difference/max": 0.21068839682266116, + "sampling/sampling_logp_difference/mean": 0.003861865133512765, + "step": 4290, + "step_time": 4.200741035968531, + "student/entropy": 0.057226090754071875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.6, + "completions/max_terminated_length": 5.6, + "completions/mean_length": 1.0688889185587565, + "completions/mean_terminated_length": 1.0688889185587565, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04759168433956802, + "epoch": 0.16405270952796872, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.995176315307617, + "kd/policy_loss": -2.995864407221476, + "kd/rkl_advantage_mean": 54.21446385383606, + "kd/rkl_advantage_p95": 73.02457580566406, + "kd/rkl_advantage_std": 30.595786728461583, + "kd/ssd_loss": 0.0, + "learning_rate": 8.359852656362738e-07, + "loss": -11.983457438151042, + "num_tokens": 71207528.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0477712194124857, + "sampling/importance_sampling_ratio/mean": 1.0001188476880392, + "sampling/importance_sampling_ratio/min": 0.8711932539939881, + "sampling/sampling_logp_difference/max": 0.16061774543486534, + "sampling/sampling_logp_difference/mean": 0.0031748994758042195, + "step": 4320, + "step_time": 4.332407055403261, + "student/entropy": 0.04759168433956802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.8, + "completions/max_terminated_length": 5.8, + "completions/mean_length": 1.0488889257113139, + "completions/mean_terminated_length": 1.0488889257113139, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.046266807119051616, + "epoch": 0.16519196445524628, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 34.90183639526367, + "kd/policy_loss": -2.393159282207489, + "kd/rkl_advantage_mean": 57.66414521535238, + "kd/rkl_advantage_p95": 84.17878561814626, + "kd/rkl_advantage_std": 38.048172841469444, + "kd/ssd_loss": 0.0, + "learning_rate": 8.348460107089963e-07, + "loss": -9.572637939453125, + "num_tokens": 71511368.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.018747099240621, + "sampling/importance_sampling_ratio/mean": 0.9997973004976909, + "sampling/importance_sampling_ratio/min": 0.8585630526145299, + "sampling/sampling_logp_difference/max": 0.17853879711280266, + "sampling/sampling_logp_difference/mean": 0.0029542325627213966, + "step": 4350, + "step_time": 4.233665303482364, + "student/entropy": 0.046266807119051616 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.266666666666667, + "completions/max_terminated_length": 5.266666666666667, + "completions/mean_length": 1.0527778148651123, + "completions/mean_terminated_length": 1.0527778148651123, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04171726385441919, + "epoch": 0.16633121938252382, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.31659698486328, + "kd/policy_loss": -1.567757433652878, + "kd/rkl_advantage_mean": 49.42694218953451, + "kd/rkl_advantage_p95": 72.30776476860046, + "kd/rkl_advantage_std": 33.636887853344284, + "kd/ssd_loss": 0.0, + "learning_rate": 8.337067557817187e-07, + "loss": -6.271030680338542, + "num_tokens": 71803462.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0545886834462483, + "sampling/importance_sampling_ratio/mean": 1.0002439677715302, + "sampling/importance_sampling_ratio/min": 0.8931760489940643, + "sampling/sampling_logp_difference/max": 0.15801963680423797, + "sampling/sampling_logp_difference/mean": 0.0025452677354527017, + "step": 4380, + "step_time": 4.197149344754871, + "student/entropy": 0.04171726385441919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.066666666666666, + "completions/max_terminated_length": 7.066666666666666, + "completions/mean_length": 1.0738889257113138, + "completions/mean_terminated_length": 1.0738889257113138, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04137811250984669, + "epoch": 0.1674704743098014, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.453168869018555, + "kd/policy_loss": -4.363157256444295, + "kd/rkl_advantage_mean": 72.10603267351786, + "kd/rkl_advantage_p95": 95.80186867713928, + "kd/rkl_advantage_std": 39.093771332502364, + "kd/ssd_loss": 0.0, + "learning_rate": 8.325675008544411e-07, + "loss": -17.452630615234376, + "num_tokens": 72113216.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0201854666074117, + "sampling/importance_sampling_ratio/mean": 0.9999276498953501, + "sampling/importance_sampling_ratio/min": 0.9080293258031209, + "sampling/sampling_logp_difference/max": 0.11491688052192331, + "sampling/sampling_logp_difference/mean": 0.002239299224068721, + "step": 4410, + "step_time": 4.267963661443598, + "student/entropy": 0.04137811250984669 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.0, + "completions/max_terminated_length": 5.0, + "completions/mean_length": 1.0677778085072835, + "completions/mean_terminated_length": 1.0677778085072835, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0627345375251025, + "epoch": 0.16860972923707895, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 53.28759002685547, + "kd/policy_loss": -2.4358747323354084, + "kd/rkl_advantage_mean": 45.4729723294576, + "kd/rkl_advantage_p95": 62.49585240681966, + "kd/rkl_advantage_std": 26.182713973522187, + "kd/ssd_loss": 0.0, + "learning_rate": 8.314282459271636e-07, + "loss": -9.743498738606771, + "num_tokens": 72415276.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0156100710233054, + "sampling/importance_sampling_ratio/mean": 1.0001940528551738, + "sampling/importance_sampling_ratio/min": 0.8846396565437317, + "sampling/sampling_logp_difference/max": 0.13996854273912807, + "sampling/sampling_logp_difference/mean": 0.0032754788427458454, + "step": 4440, + "step_time": 4.2593546497751955, + "student/entropy": 0.0627345375251025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.333333333333333, + "completions/max_terminated_length": 6.333333333333333, + "completions/mean_length": 1.071666701634725, + "completions/mean_terminated_length": 1.071666701634725, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0562937519668291, + "epoch": 0.16974898416435652, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.726605415344238, + "kd/policy_loss": -4.118077951669693, + "kd/rkl_advantage_mean": 63.83063133557638, + "kd/rkl_advantage_p95": 84.67348261674245, + "kd/rkl_advantage_std": 33.9131801913182, + "kd/ssd_loss": 0.0, + "learning_rate": 8.302889909998861e-07, + "loss": -16.47231241861979, + "num_tokens": 72714622.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0468890468279521, + "sampling/importance_sampling_ratio/mean": 1.0001954933007557, + "sampling/importance_sampling_ratio/min": 0.8711527705192565, + "sampling/sampling_logp_difference/max": 0.15798428148652116, + "sampling/sampling_logp_difference/mean": 0.0031767818378284573, + "step": 4470, + "step_time": 4.278858196952691, + "student/entropy": 0.0562937519668291 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.666666666666667, + "completions/max_terminated_length": 6.666666666666667, + "completions/mean_length": 1.0644444863001505, + "completions/mean_terminated_length": 1.0644444863001505, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03510504195777078, + "epoch": 0.17088823909163406, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.373793601989746, + "kd/policy_loss": -3.703026392062505, + "kd/rkl_advantage_mean": 67.6962784131368, + "kd/rkl_advantage_p95": 95.3674108505249, + "kd/rkl_advantage_std": 40.154692181944846, + "kd/ssd_loss": 0.0, + "learning_rate": 8.291497360726085e-07, + "loss": -14.812109375, + "num_tokens": 73024070.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0064730683962504, + "sampling/importance_sampling_ratio/mean": 0.9991992056369782, + "sampling/importance_sampling_ratio/min": 0.8588071078062057, + "sampling/sampling_logp_difference/max": 0.14164675392676146, + "sampling/sampling_logp_difference/mean": 0.0025252900280368826, + "step": 4500, + "step_time": 4.346761031386753, + "student/entropy": 0.03510504195777078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.045555587609609, + "completions/mean_terminated_length": 1.045555587609609, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0363381411259373, + "epoch": 0.17202749401891163, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.76785659790039, + "kd/policy_loss": -1.9521043300628662, + "kd/rkl_advantage_mean": 59.537214279174805, + "kd/rkl_advantage_p95": 84.6541923125585, + "kd/rkl_advantage_std": 38.90075854261716, + "kd/ssd_loss": 0.0, + "learning_rate": 8.280104811453309e-07, + "loss": -7.808416748046875, + "num_tokens": 73331642.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0059989134470622, + "sampling/importance_sampling_ratio/mean": 0.9999081869920095, + "sampling/importance_sampling_ratio/min": 0.9004574457804362, + "sampling/sampling_logp_difference/max": 0.11246228381060064, + "sampling/sampling_logp_difference/mean": 0.0015678792357599984, + "step": 4530, + "step_time": 4.2675040255843975, + "student/entropy": 0.0363381411259373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.866666666666666, + "completions/max_terminated_length": 5.866666666666666, + "completions/mean_length": 1.055833359559377, + "completions/mean_terminated_length": 1.055833359559377, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03630283460952342, + "epoch": 0.1731667489461892, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.474331855773926, + "kd/policy_loss": -2.130153489112854, + "kd/rkl_advantage_mean": 50.100834639867145, + "kd/rkl_advantage_p95": 67.43269064426423, + "kd/rkl_advantage_std": 28.73449390232563, + "kd/ssd_loss": 0.0, + "learning_rate": 8.268712262180534e-07, + "loss": -8.52061258951823, + "num_tokens": 73634203.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0014260371526083, + "sampling/importance_sampling_ratio/mean": 0.999843015273412, + "sampling/importance_sampling_ratio/min": 0.9211250444253286, + "sampling/sampling_logp_difference/max": 0.0848245145054534, + "sampling/sampling_logp_difference/mean": 0.0014345342574718718, + "step": 4560, + "step_time": 4.239354870274353, + "student/entropy": 0.03630283460952342 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.6, + "completions/max_terminated_length": 5.6, + "completions/mean_length": 1.059166701634725, + "completions/mean_terminated_length": 1.059166701634725, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04980190501858791, + "epoch": 0.17430600387346676, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 132.18727111816406, + "kd/policy_loss": -2.5901207268238067, + "kd/rkl_advantage_mean": 51.56399463017782, + "kd/rkl_advantage_p95": 72.58283263047537, + "kd/rkl_advantage_std": 31.178386038541795, + "kd/ssd_loss": 0.0, + "learning_rate": 8.257319712907757e-07, + "loss": -10.360483805338541, + "num_tokens": 73944768.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0470334529876708, + "sampling/importance_sampling_ratio/mean": 1.000240421295166, + "sampling/importance_sampling_ratio/min": 0.8867050687472026, + "sampling/sampling_logp_difference/max": 0.15447552526990574, + "sampling/sampling_logp_difference/mean": 0.002512716845376417, + "step": 4590, + "step_time": 4.247110674239229, + "student/entropy": 0.04980190501858791 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.07861115137736, + "completions/mean_terminated_length": 1.07861115137736, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.046277839705968896, + "epoch": 0.17544525880074432, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.510760307312012, + "kd/policy_loss": -3.797263342142105, + "kd/rkl_advantage_mean": 63.372486639022824, + "kd/rkl_advantage_p95": 84.3179038921992, + "kd/rkl_advantage_std": 34.2785123894612, + "kd/ssd_loss": 0.0, + "learning_rate": 8.245927163634982e-07, + "loss": -15.18905029296875, + "num_tokens": 74257395.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.016052532196045, + "sampling/importance_sampling_ratio/mean": 0.9999736229578654, + "sampling/importance_sampling_ratio/min": 0.9085243523120881, + "sampling/sampling_logp_difference/max": 0.11077149634559949, + "sampling/sampling_logp_difference/mean": 0.002038025204092264, + "step": 4620, + "step_time": 4.387012078915722, + "student/entropy": 0.046277839705968896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.4, + "completions/max_terminated_length": 5.4, + "completions/mean_length": 1.0566666920979817, + "completions/mean_terminated_length": 1.0566666920979817, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04030392970889807, + "epoch": 0.17658451372802186, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 35.179691314697266, + "kd/policy_loss": -2.0442424535751345, + "kd/rkl_advantage_mean": 48.82112323443095, + "kd/rkl_advantage_p95": 67.28234614531199, + "kd/rkl_advantage_std": 29.501599982380867, + "kd/ssd_loss": 0.0, + "learning_rate": 8.234534614362207e-07, + "loss": -8.176968383789063, + "num_tokens": 74559399.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.001843770345052, + "sampling/importance_sampling_ratio/mean": 0.999974650144577, + "sampling/importance_sampling_ratio/min": 0.9271428088347117, + "sampling/sampling_logp_difference/max": 0.07974145063975205, + "sampling/sampling_logp_difference/mean": 0.0015700901402548575, + "step": 4650, + "step_time": 4.29440127943332, + "student/entropy": 0.04030392970889807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.933333333333334, + "completions/max_terminated_length": 5.933333333333334, + "completions/mean_length": 1.0588889241218566, + "completions/mean_terminated_length": 1.0588889241218566, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04894599212954442, + "epoch": 0.17772376865529943, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 31.066194534301758, + "kd/policy_loss": -2.501105244954427, + "kd/rkl_advantage_mean": 53.47761181195577, + "kd/rkl_advantage_p95": 72.87078881263733, + "kd/rkl_advantage_std": 31.0211098442475, + "kd/ssd_loss": 0.0, + "learning_rate": 8.223142065089432e-07, + "loss": -10.004421997070313, + "num_tokens": 74858667.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0131098628044128, + "sampling/importance_sampling_ratio/mean": 1.000153907140096, + "sampling/importance_sampling_ratio/min": 0.908270005385081, + "sampling/sampling_logp_difference/max": 0.10284527968615294, + "sampling/sampling_logp_difference/mean": 0.0017695543259227027, + "step": 4680, + "step_time": 4.245231100230012, + "student/entropy": 0.04894599212954442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.733333333333333, + "completions/max_terminated_length": 5.733333333333333, + "completions/mean_length": 1.0450000286102294, + "completions/mean_terminated_length": 1.0450000286102294, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0347319349180907, + "epoch": 0.178863023582577, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 122.98515319824219, + "kd/policy_loss": -1.1752900242805482, + "kd/rkl_advantage_mean": 50.725765546162926, + "kd/rkl_advantage_p95": 72.89098315238952, + "kd/rkl_advantage_std": 34.24762896001339, + "kd/ssd_loss": 0.0, + "learning_rate": 8.211749515816655e-07, + "loss": -4.701160685221354, + "num_tokens": 75164333.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.001846194267273, + "sampling/importance_sampling_ratio/mean": 1.0000168164571126, + "sampling/importance_sampling_ratio/min": 0.9323260068893433, + "sampling/sampling_logp_difference/max": 0.07829835050894568, + "sampling/sampling_logp_difference/mean": 0.0011904623213922604, + "step": 4710, + "step_time": 4.268175669344298, + "student/entropy": 0.0347319349180907 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.666666666666667, + "completions/max_terminated_length": 6.666666666666667, + "completions/mean_length": 1.071666701634725, + "completions/mean_terminated_length": 1.071666701634725, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04454746798922618, + "epoch": 0.18000227850985456, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.5879487991333, + "kd/policy_loss": -4.195059055089951, + "kd/rkl_advantage_mean": 63.16865458488464, + "kd/rkl_advantage_p95": 84.30003401438395, + "kd/rkl_advantage_std": 32.99000370403131, + "kd/ssd_loss": 0.0, + "learning_rate": 8.20035696654388e-07, + "loss": -16.780240885416667, + "num_tokens": 75467535.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0308042446772256, + "sampling/importance_sampling_ratio/mean": 1.0001003940900166, + "sampling/importance_sampling_ratio/min": 0.9068857997655868, + "sampling/sampling_logp_difference/max": 0.10320779792964459, + "sampling/sampling_logp_difference/mean": 0.002075177499985633, + "step": 4740, + "step_time": 4.233451749656039, + "student/entropy": 0.04454746798922618 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0616666992505392, + "completions/mean_terminated_length": 1.0616666992505392, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07561095440760254, + "epoch": 0.1811415334371321, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 54.896392822265625, + "kd/policy_loss": -1.9011352936426797, + "kd/rkl_advantage_mean": 50.110746161142984, + "kd/rkl_advantage_p95": 68.10093814531962, + "kd/rkl_advantage_std": 29.667811278502146, + "kd/ssd_loss": 0.0, + "learning_rate": 8.188964417271105e-07, + "loss": -7.604542032877604, + "num_tokens": 75774213.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.018564188480377, + "sampling/importance_sampling_ratio/mean": 1.0003248751163483, + "sampling/importance_sampling_ratio/min": 0.8951707720756531, + "sampling/sampling_logp_difference/max": 0.1238647123798728, + "sampling/sampling_logp_difference/mean": 0.002656753727933392, + "step": 4770, + "step_time": 4.246693028028433, + "student/entropy": 0.07561095440760254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.033333333333333, + "completions/max_terminated_length": 7.033333333333333, + "completions/mean_length": 1.0758333722750346, + "completions/mean_terminated_length": 1.0758333722750346, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.050013332689801855, + "epoch": 0.18228078836440967, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 7.687243461608887, + "kd/policy_loss": -3.213956646124522, + "kd/rkl_advantage_mean": 62.99736205736796, + "kd/rkl_advantage_p95": 84.54342704614004, + "kd/rkl_advantage_std": 35.12233750273784, + "kd/ssd_loss": 0.0, + "learning_rate": 8.177571867998328e-07, + "loss": -12.855826822916667, + "num_tokens": 76076718.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0301270445187887, + "sampling/importance_sampling_ratio/mean": 0.9995902160803477, + "sampling/importance_sampling_ratio/min": 0.869618704666694, + "sampling/sampling_logp_difference/max": 0.22232401272437224, + "sampling/sampling_logp_difference/mean": 0.002699116314761341, + "step": 4800, + "step_time": 4.283028240377704, + "student/entropy": 0.050013332689801855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.3, + "completions/max_terminated_length": 6.3, + "completions/mean_length": 1.0697222590446471, + "completions/mean_terminated_length": 1.0697222590446471, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06615319782868027, + "epoch": 0.18342004329168723, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 30.379575729370117, + "kd/policy_loss": -3.45071285367012, + "kd/rkl_advantage_mean": 58.3384441057841, + "kd/rkl_advantage_p95": 78.47484524250031, + "kd/rkl_advantage_std": 31.72957919041316, + "kd/ssd_loss": 0.0, + "learning_rate": 8.166179318725553e-07, + "loss": -13.802852376302083, + "num_tokens": 76375329.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0712903340657551, + "sampling/importance_sampling_ratio/mean": 0.9998658974965413, + "sampling/importance_sampling_ratio/min": 0.8880128502845764, + "sampling/sampling_logp_difference/max": 0.1406065429793671, + "sampling/sampling_logp_difference/mean": 0.0025472882494796067, + "step": 4830, + "step_time": 4.256682015613963, + "student/entropy": 0.06615319782868027 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.7, + "completions/max_terminated_length": 5.7, + "completions/mean_length": 1.0497222582499186, + "completions/mean_terminated_length": 1.0497222582499186, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.044665668780604996, + "epoch": 0.1845592982189648, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.28470230102539, + "kd/policy_loss": -1.859944764773051, + "kd/rkl_advantage_mean": 52.336365413665774, + "kd/rkl_advantage_p95": 73.01951431433359, + "kd/rkl_advantage_std": 33.10384773711363, + "kd/ssd_loss": 0.0, + "learning_rate": 8.154786769452778e-07, + "loss": -7.439777628580729, + "num_tokens": 76683180.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0127031644185385, + "sampling/importance_sampling_ratio/mean": 0.9994442105293274, + "sampling/importance_sampling_ratio/min": 0.9151342590649922, + "sampling/sampling_logp_difference/max": 0.0989134727511555, + "sampling/sampling_logp_difference/mean": 0.0015230048641872902, + "step": 4860, + "step_time": 4.175651274470146, + "student/entropy": 0.044665668780604996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.233333333333333, + "completions/max_terminated_length": 6.233333333333333, + "completions/mean_length": 1.078611147403717, + "completions/mean_terminated_length": 1.078611147403717, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06679921650017301, + "epoch": 0.18569855314624237, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.557300567626953, + "kd/policy_loss": -4.95642394622167, + "kd/rkl_advantage_mean": 64.22696754137675, + "kd/rkl_advantage_p95": 84.97770569324493, + "kd/rkl_advantage_std": 31.962393407026926, + "kd/ssd_loss": 0.0, + "learning_rate": 8.143394220180002e-07, + "loss": -19.82569580078125, + "num_tokens": 76995143.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.152049857378006, + "sampling/importance_sampling_ratio/mean": 0.9997783442338307, + "sampling/importance_sampling_ratio/min": 0.9182203610738119, + "sampling/sampling_logp_difference/max": 0.18794471224149067, + "sampling/sampling_logp_difference/mean": 0.004425786039792001, + "step": 4890, + "step_time": 4.307143643533346, + "student/entropy": 0.06679921650017301 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.933333333333334, + "completions/max_terminated_length": 5.933333333333334, + "completions/mean_length": 1.0922222534815471, + "completions/mean_terminated_length": 1.0922222534815471, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07338465324913462, + "epoch": 0.1868378080735199, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 37.26102066040039, + "kd/policy_loss": -4.450540828704834, + "kd/rkl_advantage_mean": 53.43354345957438, + "kd/rkl_advantage_p95": 67.87010745207469, + "kd/rkl_advantage_std": 23.81737932463487, + "kd/ssd_loss": 0.0, + "learning_rate": 8.132001670907226e-07, + "loss": -17.802164713541668, + "num_tokens": 77311787.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0174981117248536, + "sampling/importance_sampling_ratio/mean": 0.9997569104035695, + "sampling/importance_sampling_ratio/min": 0.8681246675550938, + "sampling/sampling_logp_difference/max": 0.139996293031921, + "sampling/sampling_logp_difference/mean": 0.0030410533673906077, + "step": 4920, + "step_time": 4.391375001570365, + "student/entropy": 0.07338465324913462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.2, + "completions/max_terminated_length": 6.2, + "completions/mean_length": 1.066111139456431, + "completions/mean_terminated_length": 1.066111139456431, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054738578572869304, + "epoch": 0.18797706300079747, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.219501495361328, + "kd/policy_loss": -2.858530088265737, + "kd/rkl_advantage_mean": 58.140669918060304, + "kd/rkl_advantage_p95": 78.97710332870483, + "kd/rkl_advantage_std": 33.748691245913506, + "kd/ssd_loss": 0.0, + "learning_rate": 8.120609121634451e-07, + "loss": -11.43411865234375, + "num_tokens": 77623577.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0142114718755086, + "sampling/importance_sampling_ratio/mean": 0.9997902890046437, + "sampling/importance_sampling_ratio/min": 0.8951541393995285, + "sampling/sampling_logp_difference/max": 0.13369233291596175, + "sampling/sampling_logp_difference/mean": 0.002156488176357622, + "step": 4950, + "step_time": 4.36655135951393, + "student/entropy": 0.054738578572869304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0416666944821675, + "completions/mean_terminated_length": 1.0416666944821675, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04324912177398801, + "epoch": 0.18911631792807504, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.592700958251953, + "kd/policy_loss": -1.2204022963841756, + "kd/rkl_advantage_mean": 54.59735217094421, + "kd/rkl_advantage_p95": 78.50172719955444, + "kd/rkl_advantage_std": 37.09494970192512, + "kd/ssd_loss": 0.0, + "learning_rate": 8.109216572361674e-07, + "loss": -4.881608072916666, + "num_tokens": 77922823.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0098113457361857, + "sampling/importance_sampling_ratio/mean": 1.0000348289807637, + "sampling/importance_sampling_ratio/min": 0.9266339699427287, + "sampling/sampling_logp_difference/max": 0.08446592135975758, + "sampling/sampling_logp_difference/mean": 0.0012337980733718723, + "step": 4980, + "step_time": 4.151770542185599, + "student/entropy": 0.04324912177398801 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.4, + "completions/max_terminated_length": 5.4, + "completions/mean_length": 1.0394444664319356, + "completions/mean_terminated_length": 1.0394444664319356, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03710161585671206, + "epoch": 0.1902555728553526, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.260652542114258, + "kd/policy_loss": -0.49464460213979083, + "kd/rkl_advantage_mean": 43.13044050534566, + "kd/rkl_advantage_p95": 62.271996100743614, + "kd/rkl_advantage_std": 30.064015853405, + "kd/ssd_loss": 0.0, + "learning_rate": 8.097824023088899e-07, + "loss": -1.9785784403483073, + "num_tokens": 78246909.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0307181398073832, + "sampling/importance_sampling_ratio/mean": 1.0002431452274323, + "sampling/importance_sampling_ratio/min": 0.9455472449461619, + "sampling/sampling_logp_difference/max": 0.08121018313492338, + "sampling/sampling_logp_difference/mean": 0.0011122962197987363, + "step": 5010, + "step_time": 4.313175123619536, + "student/entropy": 0.03710161585671206 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.466666666666667, + "completions/max_terminated_length": 7.466666666666667, + "completions/mean_length": 1.0733333746592204, + "completions/mean_terminated_length": 1.0733333746592204, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048760127183049916, + "epoch": 0.19139482778263017, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.808658599853516, + "kd/policy_loss": -4.423288838068644, + "kd/rkl_advantage_mean": 73.42996575037638, + "kd/rkl_advantage_p95": 96.04296062787374, + "kd/rkl_advantage_std": 38.435722361008324, + "kd/ssd_loss": 0.0, + "learning_rate": 8.086431473816125e-07, + "loss": -17.69315388997396, + "num_tokens": 78551605.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0028385281562806, + "sampling/importance_sampling_ratio/mean": 0.9996099889278411, + "sampling/importance_sampling_ratio/min": 0.8970501522223154, + "sampling/sampling_logp_difference/max": 0.10593239776790142, + "sampling/sampling_logp_difference/mean": 0.0018699002568610013, + "step": 5040, + "step_time": 4.154392660413093, + "student/entropy": 0.048760127183049916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.166666666666667, + "completions/max_terminated_length": 4.166666666666667, + "completions/mean_length": 1.0336111307144165, + "completions/mean_terminated_length": 1.0336111307144165, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04312918738772472, + "epoch": 0.1925340827099077, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 30.020618438720703, + "kd/policy_loss": 0.3867712259292603, + "kd/rkl_advantage_mean": 29.153810040156046, + "kd/rkl_advantage_p95": 44.97679618994395, + "kd/rkl_advantage_std": 23.59046223461628, + "kd/ssd_loss": 0.0, + "learning_rate": 8.075038924543348e-07, + "loss": 1.5470839182535807, + "num_tokens": 78866598.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0349544843037923, + "sampling/importance_sampling_ratio/mean": 1.0000715692838034, + "sampling/importance_sampling_ratio/min": 0.9168531874815623, + "sampling/sampling_logp_difference/max": 0.10618044547736645, + "sampling/sampling_logp_difference/mean": 0.001567085397740205, + "step": 5070, + "step_time": 4.228612943265277, + "student/entropy": 0.04312918738772472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.366666666666666, + "completions/max_terminated_length": 6.366666666666666, + "completions/mean_length": 1.065833361943563, + "completions/mean_terminated_length": 1.065833361943563, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05625016742075483, + "epoch": 0.19367333763718528, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.132365226745605, + "kd/policy_loss": -3.0649157881736757, + "kd/rkl_advantage_mean": 55.83287051518758, + "kd/rkl_advantage_p95": 73.18997627894083, + "kd/rkl_advantage_std": 29.485384200016657, + "kd/ssd_loss": 0.0, + "learning_rate": 8.063646375270573e-07, + "loss": -12.259665934244792, + "num_tokens": 79174851.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0536388278007507, + "sampling/importance_sampling_ratio/mean": 1.0000763376553854, + "sampling/importance_sampling_ratio/min": 0.8864040106534958, + "sampling/sampling_logp_difference/max": 0.15708742939556639, + "sampling/sampling_logp_difference/mean": 0.0024248735775472597, + "step": 5100, + "step_time": 4.207001102284994, + "student/entropy": 0.05625016742075483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.933333333333334, + "completions/max_terminated_length": 6.933333333333334, + "completions/mean_length": 1.0777778188387552, + "completions/mean_terminated_length": 1.0777778188387552, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0526521069308122, + "epoch": 0.19481259256446284, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 37.80744552612305, + "kd/policy_loss": -4.195271501938502, + "kd/rkl_advantage_mean": 67.59555735588074, + "kd/rkl_advantage_p95": 90.1881206591924, + "kd/rkl_advantage_std": 35.85488676627477, + "kd/ssd_loss": 0.0, + "learning_rate": 8.052253825997797e-07, + "loss": -16.781086222330728, + "num_tokens": 79478715.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0408733407656352, + "sampling/importance_sampling_ratio/mean": 1.0000518878300986, + "sampling/importance_sampling_ratio/min": 0.9116465111573537, + "sampling/sampling_logp_difference/max": 0.12308547313635548, + "sampling/sampling_logp_difference/mean": 0.0023140599582499514, + "step": 5130, + "step_time": 4.228477320668754, + "student/entropy": 0.0526521069308122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0688889185587565, + "completions/mean_terminated_length": 1.0688889185587565, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04574007665117582, + "epoch": 0.1959518474917404, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.211993217468262, + "kd/policy_loss": -3.39379004240036, + "kd/rkl_advantage_mean": 59.3453578154246, + "kd/rkl_advantage_p95": 79.00627968311309, + "kd/rkl_advantage_std": 32.175557765364644, + "kd/ssd_loss": 0.0, + "learning_rate": 8.040861276725022e-07, + "loss": -13.575162760416667, + "num_tokens": 79789651.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0356409708658854, + "sampling/importance_sampling_ratio/mean": 0.9996677001317342, + "sampling/importance_sampling_ratio/min": 0.8996858537197113, + "sampling/sampling_logp_difference/max": 0.12270304830744863, + "sampling/sampling_logp_difference/mean": 0.0021155958946716662, + "step": 5160, + "step_time": 4.163230782665778, + "student/entropy": 0.04574007665117582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.413793103448276, + "completions/max_terminated_length": 5.413793103448276, + "completions/mean_length": 1.050000030418922, + "completions/mean_terminated_length": 1.050000030418922, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.042795189656317234, + "epoch": 0.19709110241901795, + "eval/gt_acc_batch": 0.0002873563368258805, + "frac_reward_zero_std": 1.0, + "grad_norm": 67.42792510986328, + "kd/policy_loss": -1.0834901723368415, + "kd/rkl_advantage_mean": 46.668717795404895, + "kd/rkl_advantage_p95": 70.1473652001085, + "kd/rkl_advantage_std": 33.001504190009214, + "kd/ssd_loss": 0.0, + "learning_rate": 8.029468727452245e-07, + "loss": -4.189497375488282, + "num_tokens": 80084353.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0002873563368258805, + "rewards/gt_logging_reward/std": 0.003147830993964754, + "sampling/importance_sampling_ratio/max": 1.0793262802321335, + "sampling/importance_sampling_ratio/mean": 1.000118378935189, + "sampling/importance_sampling_ratio/min": 0.8970274760805327, + "sampling/sampling_logp_difference/max": 0.1474668904657251, + "sampling/sampling_logp_difference/mean": 0.0019784558028512604, + "step": 5190, + "step_time": 4.064978030541291, + "student/entropy": 0.042795189656317234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.733333333333333, + "completions/max_terminated_length": 6.733333333333333, + "completions/mean_length": 1.0766667008399964, + "completions/mean_terminated_length": 1.0766667008399964, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05837061889469623, + "epoch": 0.19823035734629552, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 73.84103393554688, + "kd/policy_loss": -4.043649860223135, + "kd/rkl_advantage_mean": 63.09631614685058, + "kd/rkl_advantage_p95": 84.11951931317647, + "kd/rkl_advantage_std": 33.14278574486573, + "kd/ssd_loss": 0.0, + "learning_rate": 8.01807617817947e-07, + "loss": -16.174601236979168, + "num_tokens": 80393589.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0300535003344218, + "sampling/importance_sampling_ratio/mean": 1.0000910858313243, + "sampling/importance_sampling_ratio/min": 0.9180967330932617, + "sampling/sampling_logp_difference/max": 0.11092571686021983, + "sampling/sampling_logp_difference/mean": 0.002127611982480933, + "step": 5220, + "step_time": 4.205151462345384, + "student/entropy": 0.05837061889469623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0555555860201518, + "completions/mean_terminated_length": 1.0555555860201518, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049105147800097865, + "epoch": 0.19936961227357308, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.507579803466797, + "kd/policy_loss": -1.8961923201878865, + "kd/rkl_advantage_mean": 53.31323351860046, + "kd/rkl_advantage_p95": 73.89625124136607, + "kd/rkl_advantage_std": 32.92277084390322, + "kd/ssd_loss": 0.0, + "learning_rate": 8.006683628906695e-07, + "loss": -7.584768676757813, + "num_tokens": 80717357.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0305044571558635, + "sampling/importance_sampling_ratio/mean": 1.0001365224520364, + "sampling/importance_sampling_ratio/min": 0.9157132764657339, + "sampling/sampling_logp_difference/max": 0.08906351650754611, + "sampling/sampling_logp_difference/mean": 0.0015947198854216063, + "step": 5250, + "step_time": 4.301249528086434, + "student/entropy": 0.049105147800097865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0688889225323994, + "completions/mean_terminated_length": 1.0688889225323994, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06375496750697493, + "epoch": 0.20050886720085065, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.578420639038086, + "kd/policy_loss": -3.164316272735596, + "kd/rkl_advantage_mean": 55.43263738950093, + "kd/rkl_advantage_p95": 73.32188087304434, + "kd/rkl_advantage_std": 29.71020310819149, + "kd/ssd_loss": 0.0, + "learning_rate": 7.995291079633919e-07, + "loss": -12.65726318359375, + "num_tokens": 81025317.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.024422780672709, + "sampling/importance_sampling_ratio/mean": 1.0000497678915659, + "sampling/importance_sampling_ratio/min": 0.8922208239634831, + "sampling/sampling_logp_difference/max": 0.12497918205335737, + "sampling/sampling_logp_difference/mean": 0.0026437725968814145, + "step": 5280, + "step_time": 4.1683544656028975, + "student/entropy": 0.06375496750697493 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.3, + "completions/max_terminated_length": 5.3, + "completions/mean_length": 1.0497222542762756, + "completions/mean_terminated_length": 1.0497222542762756, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049922934298714, + "epoch": 0.20164812212812822, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.460662841796875, + "kd/policy_loss": -1.3434693535168967, + "kd/rkl_advantage_mean": 46.64699444770813, + "kd/rkl_advantage_p95": 67.39450543721517, + "kd/rkl_advantage_std": 31.44289852877458, + "kd/ssd_loss": 0.0, + "learning_rate": 7.983898530361144e-07, + "loss": -5.373877970377604, + "num_tokens": 81332992.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.008789030710856, + "sampling/importance_sampling_ratio/mean": 0.9998460094134013, + "sampling/importance_sampling_ratio/min": 0.9193839013576508, + "sampling/sampling_logp_difference/max": 0.08043841544228296, + "sampling/sampling_logp_difference/mean": 0.0016163403847410033, + "step": 5310, + "step_time": 4.197696909677082, + "student/entropy": 0.049922934298714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.566666666666666, + "completions/max_terminated_length": 7.566666666666666, + "completions/mean_length": 1.0963889280954997, + "completions/mean_terminated_length": 1.0963889280954997, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06847344736258189, + "epoch": 0.20278737705540575, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.13328742980957, + "kd/policy_loss": -5.205905000368754, + "kd/rkl_advantage_mean": 75.94373304049174, + "kd/rkl_advantage_p95": 101.06263875166574, + "kd/rkl_advantage_std": 39.66418964664141, + "kd/ssd_loss": 0.0, + "learning_rate": 7.972505981088368e-07, + "loss": -20.823624674479166, + "num_tokens": 81637555.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0453548749287924, + "sampling/importance_sampling_ratio/mean": 0.999948384364446, + "sampling/importance_sampling_ratio/min": 0.8798860301574071, + "sampling/sampling_logp_difference/max": 0.13245252290119727, + "sampling/sampling_logp_difference/mean": 0.002484991500386968, + "step": 5340, + "step_time": 4.190646033346032, + "student/entropy": 0.06847344736258189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.633333333333334, + "completions/max_terminated_length": 5.633333333333334, + "completions/mean_length": 1.0447222550710042, + "completions/mean_terminated_length": 1.0447222550710042, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.044423453013102215, + "epoch": 0.20392663198268332, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.656593322753906, + "kd/policy_loss": -1.976318468650182, + "kd/rkl_advantage_mean": 53.2088853041331, + "kd/rkl_advantage_p95": 78.18560759226482, + "kd/rkl_advantage_std": 35.73640348464251, + "kd/ssd_loss": 0.0, + "learning_rate": 7.961113431815592e-07, + "loss": -7.905275472005209, + "num_tokens": 81951260.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0380377491315207, + "sampling/importance_sampling_ratio/mean": 1.0000479340553283, + "sampling/importance_sampling_ratio/min": 0.9256175299485524, + "sampling/sampling_logp_difference/max": 0.11364541635848582, + "sampling/sampling_logp_difference/mean": 0.0019264413208778326, + "step": 5370, + "step_time": 4.213949014502577, + "student/entropy": 0.044423453013102215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.166666666666667, + "completions/max_terminated_length": 6.166666666666667, + "completions/mean_length": 1.0691667079925538, + "completions/mean_terminated_length": 1.0691667079925538, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052649016057451564, + "epoch": 0.2050658869099609, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 30.936538696289062, + "kd/policy_loss": -3.1576256414254504, + "kd/rkl_advantage_mean": 61.020155906677246, + "kd/rkl_advantage_p95": 89.85256356398264, + "kd/rkl_advantage_std": 38.87634725123644, + "kd/ssd_loss": 0.0, + "learning_rate": 7.949720882542816e-07, + "loss": -12.630501302083333, + "num_tokens": 82264477.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0421142856280008, + "sampling/importance_sampling_ratio/mean": 0.9998086055119833, + "sampling/importance_sampling_ratio/min": 0.888101319471995, + "sampling/sampling_logp_difference/max": 0.16512332243534425, + "sampling/sampling_logp_difference/mean": 0.002500941584003158, + "step": 5400, + "step_time": 4.196159665216692, + "student/entropy": 0.052649016057451564 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.241379310344827, + "completions/max_terminated_length": 5.241379310344827, + "completions/mean_length": 1.0439655451939023, + "completions/mean_terminated_length": 1.0439655451939023, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04300413001328707, + "epoch": 0.20620514183723845, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 31.04174041748047, + "kd/policy_loss": -0.45156541363946323, + "kd/rkl_advantage_mean": 42.4876797939169, + "kd/rkl_advantage_p95": 64.13953021477009, + "kd/rkl_advantage_std": 31.25395963777756, + "kd/ssd_loss": 0.0, + "learning_rate": 7.938328333270042e-07, + "loss": -1.7460515340169271, + "num_tokens": 82557046.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0059865219839688, + "sampling/importance_sampling_ratio/mean": 0.9998455232587354, + "sampling/importance_sampling_ratio/min": 0.922578172437076, + "sampling/sampling_logp_difference/max": 0.08324871989416666, + "sampling/sampling_logp_difference/mean": 0.0013217813243029703, + "step": 5430, + "step_time": 4.1089811236830425, + "student/entropy": 0.04300413001328707 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.966666666666667, + "completions/max_terminated_length": 5.966666666666667, + "completions/mean_length": 1.0597222487131754, + "completions/mean_terminated_length": 1.0597222487131754, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04241873570717871, + "epoch": 0.20734439676451602, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 29.50970458984375, + "kd/policy_loss": -2.0967969059944154, + "kd/rkl_advantage_mean": 53.07876920700073, + "kd/rkl_advantage_p95": 73.18409475485484, + "kd/rkl_advantage_std": 32.122249547640486, + "kd/ssd_loss": 0.0, + "learning_rate": 7.926935783997265e-07, + "loss": -8.387186177571614, + "num_tokens": 82868509.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0233245491981506, + "sampling/importance_sampling_ratio/mean": 0.9997368494669596, + "sampling/importance_sampling_ratio/min": 0.9033040245374043, + "sampling/sampling_logp_difference/max": 0.11042057080194354, + "sampling/sampling_logp_difference/mean": 0.0016278689125707993, + "step": 5460, + "step_time": 4.281952209254571, + "student/entropy": 0.04241873570717871 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0644444823265076, + "completions/mean_terminated_length": 1.0644444823265076, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06020072170843681, + "epoch": 0.20848365169179356, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 36.945404052734375, + "kd/policy_loss": -3.1524026652177173, + "kd/rkl_advantage_mean": 61.49431988398234, + "kd/rkl_advantage_p95": 84.2138456662496, + "kd/rkl_advantage_std": 35.75029435654481, + "kd/ssd_loss": 0.0, + "learning_rate": 7.91554323472449e-07, + "loss": -12.609613037109375, + "num_tokens": 83164165.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0134039441744487, + "sampling/importance_sampling_ratio/mean": 0.9996238430341085, + "sampling/importance_sampling_ratio/min": 0.8929836571216583, + "sampling/sampling_logp_difference/max": 0.1330936240653197, + "sampling/sampling_logp_difference/mean": 0.0021657579093395424, + "step": 5490, + "step_time": 4.250835068248367, + "student/entropy": 0.06020072170843681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.066666666666666, + "completions/max_terminated_length": 6.066666666666666, + "completions/mean_length": 1.0572222550710042, + "completions/mean_terminated_length": 1.0572222550710042, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.044801000195244946, + "epoch": 0.20962290661907113, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.871898651123047, + "kd/policy_loss": -1.9599885801474253, + "kd/rkl_advantage_mean": 54.24675998687744, + "kd/rkl_advantage_p95": 78.41949752171834, + "kd/rkl_advantage_std": 35.351653578877446, + "kd/ssd_loss": 0.0, + "learning_rate": 7.904150685451714e-07, + "loss": -7.839955139160156, + "num_tokens": 83472763.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0515317956606547, + "sampling/importance_sampling_ratio/mean": 1.000583330790202, + "sampling/importance_sampling_ratio/min": 0.9405278404553731, + "sampling/sampling_logp_difference/max": 0.11267009619623422, + "sampling/sampling_logp_difference/mean": 0.0018009846632291253, + "step": 5520, + "step_time": 4.293355507527789, + "student/entropy": 0.044801000195244946 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.866666666666666, + "completions/max_terminated_length": 4.866666666666666, + "completions/mean_length": 1.0438889185587565, + "completions/mean_terminated_length": 1.0438889185587565, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0468441903591156, + "epoch": 0.2107621615463487, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 29.69837188720703, + "kd/policy_loss": -1.010741666952769, + "kd/rkl_advantage_mean": 39.34205066363017, + "kd/rkl_advantage_p95": 55.96870086193085, + "kd/rkl_advantage_std": 26.18102686305841, + "kd/ssd_loss": 0.0, + "learning_rate": 7.892758136178938e-07, + "loss": -4.042966206868489, + "num_tokens": 83778473.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.002903421719869, + "sampling/importance_sampling_ratio/mean": 1.0000228007634482, + "sampling/importance_sampling_ratio/min": 0.9277626891930898, + "sampling/sampling_logp_difference/max": 0.07985260750477513, + "sampling/sampling_logp_difference/mean": 0.0014407361527749647, + "step": 5550, + "step_time": 4.171469392674044, + "student/entropy": 0.0468441903591156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0605555891990661, + "completions/mean_terminated_length": 1.0605555891990661, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06926836020623645, + "epoch": 0.21190141647362626, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.690139770507812, + "kd/policy_loss": -2.826310368378957, + "kd/rkl_advantage_mean": 61.851794449488324, + "kd/rkl_advantage_p95": 84.448224512736, + "kd/rkl_advantage_std": 36.750126512845355, + "kd/ssd_loss": 0.0, + "learning_rate": 7.881365586906163e-07, + "loss": -11.305240885416667, + "num_tokens": 84084579.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0081241567929586, + "sampling/importance_sampling_ratio/mean": 1.000341675678889, + "sampling/importance_sampling_ratio/min": 0.9125711818536123, + "sampling/sampling_logp_difference/max": 0.09840412614867092, + "sampling/sampling_logp_difference/mean": 0.0021817314593742293, + "step": 5580, + "step_time": 4.121468156626603, + "student/entropy": 0.06926836020623645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.866666666666666, + "completions/max_terminated_length": 5.866666666666666, + "completions/mean_length": 1.0494444767634075, + "completions/mean_terminated_length": 1.0494444767634075, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048476036203404266, + "epoch": 0.2130406714009038, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 29.631223678588867, + "kd/policy_loss": -2.1643396437168123, + "kd/rkl_advantage_mean": 55.44392064412435, + "kd/rkl_advantage_p95": 78.46704782644908, + "kd/rkl_advantage_std": 35.255943567554155, + "kd/ssd_loss": 0.0, + "learning_rate": 7.869973037633387e-07, + "loss": -8.657358805338541, + "num_tokens": 84397037.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0045361995697022, + "sampling/importance_sampling_ratio/mean": 0.9996985872586568, + "sampling/importance_sampling_ratio/min": 0.9050894300142924, + "sampling/sampling_logp_difference/max": 0.09936555769915383, + "sampling/sampling_logp_difference/mean": 0.0015891845202228676, + "step": 5610, + "step_time": 4.216894849572175, + "student/entropy": 0.048476036203404266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 22.966666666666665, + "completions/max_terminated_length": 6.2, + "completions/mean_length": 1.1997222701708476, + "completions/mean_terminated_length": 1.0578104933102925, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04260063835730155, + "epoch": 0.21417992632818136, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 28.43167495727539, + "kd/policy_loss": -2.9731087068716686, + "kd/rkl_advantage_mean": 60.72676499684652, + "kd/rkl_advantage_p95": 84.22653030554453, + "kd/rkl_advantage_std": 34.300519129633905, + "kd/ssd_loss": 0.0, + "learning_rate": 7.858580488360613e-07, + "loss": -11.892434692382812, + "num_tokens": 84721452.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0733759125073752, + "sampling/importance_sampling_ratio/mean": 1.0001175343990325, + "sampling/importance_sampling_ratio/min": 0.8902041363219421, + "sampling/sampling_logp_difference/max": 0.2096131220770379, + "sampling/sampling_logp_difference/mean": 0.0025922165950760245, + "step": 5640, + "step_time": 4.402332333754748, + "student/entropy": 0.04260063835730155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.533333333333333, + "completions/max_terminated_length": 5.533333333333333, + "completions/mean_length": 1.0677778085072835, + "completions/mean_terminated_length": 1.0677778085072835, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.051887945272028446, + "epoch": 0.21531918125545893, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 319.76361083984375, + "kd/policy_loss": -3.3732807914415996, + "kd/rkl_advantage_mean": 51.02199045817057, + "kd/rkl_advantage_p95": 67.7571922938029, + "kd/rkl_advantage_std": 26.738847970962524, + "kd/ssd_loss": 0.0, + "learning_rate": 7.847187939087836e-07, + "loss": -13.493123372395834, + "num_tokens": 85033664.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0679401954015095, + "sampling/importance_sampling_ratio/mean": 1.00024311542511, + "sampling/importance_sampling_ratio/min": 0.9263803641001384, + "sampling/sampling_logp_difference/max": 0.1187409991864115, + "sampling/sampling_logp_difference/mean": 0.0021993479060862836, + "step": 5670, + "step_time": 4.214412643993273, + "student/entropy": 0.051887945272028446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.333333333333333, + "completions/max_terminated_length": 6.333333333333333, + "completions/mean_length": 1.0533333579699198, + "completions/mean_terminated_length": 1.0533333579699198, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05150811184818546, + "epoch": 0.2164584361827365, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.464168548583984, + "kd/policy_loss": -2.2975914080937705, + "kd/rkl_advantage_mean": 60.44881841341655, + "kd/rkl_advantage_p95": 84.61257944107055, + "kd/rkl_advantage_std": 37.9452646791935, + "kd/ssd_loss": 0.0, + "learning_rate": 7.835795389815061e-07, + "loss": -9.190366617838542, + "num_tokens": 85363616.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0394226392110189, + "sampling/importance_sampling_ratio/mean": 0.9997527182102204, + "sampling/importance_sampling_ratio/min": 0.9012993951638539, + "sampling/sampling_logp_difference/max": 0.13449168448957305, + "sampling/sampling_logp_difference/mean": 0.0019405696511967108, + "step": 5700, + "step_time": 4.296915061488592, + "student/entropy": 0.05150811184818546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.6, + "completions/max_terminated_length": 6.6, + "completions/mean_length": 1.0677778164545695, + "completions/mean_terminated_length": 1.0677778164545695, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04807176909719904, + "epoch": 0.21759769111001406, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.53553581237793, + "kd/policy_loss": -4.744408170382182, + "kd/rkl_advantage_mean": 70.31466418902079, + "kd/rkl_advantage_p95": 95.75569842656454, + "kd/rkl_advantage_std": 38.50505268375079, + "kd/ssd_loss": 0.0, + "learning_rate": 7.824402840542285e-07, + "loss": -18.97763671875, + "num_tokens": 85668620.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.1472041527430217, + "sampling/importance_sampling_ratio/mean": 1.0005409757296244, + "sampling/importance_sampling_ratio/min": 0.9103560904661815, + "sampling/sampling_logp_difference/max": 0.20355468452131997, + "sampling/sampling_logp_difference/mean": 0.002952793041671005, + "step": 5730, + "step_time": 4.2005157576447045, + "student/entropy": 0.04807176909719904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.9, + "completions/max_terminated_length": 4.9, + "completions/mean_length": 1.0497222503026327, + "completions/mean_terminated_length": 1.0497222503026327, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.030442102222392957, + "epoch": 0.2187369460372916, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 122.77230834960938, + "kd/policy_loss": -1.3456069250901541, + "kd/rkl_advantage_mean": 39.58093215624491, + "kd/rkl_advantage_p95": 55.79375290075938, + "kd/rkl_advantage_std": 25.172613487641016, + "kd/ssd_loss": 0.0, + "learning_rate": 7.813010291269509e-07, + "loss": -5.382428995768229, + "num_tokens": 85978887.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.019334888458252, + "sampling/importance_sampling_ratio/mean": 0.999773778518041, + "sampling/importance_sampling_ratio/min": 0.936527818441391, + "sampling/sampling_logp_difference/max": 0.07815545026678591, + "sampling/sampling_logp_difference/mean": 0.0013918188846825312, + "step": 5760, + "step_time": 4.159013295790646, + "student/entropy": 0.030442102222392957 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.2, + "completions/max_terminated_length": 7.2, + "completions/mean_length": 1.071666705608368, + "completions/mean_terminated_length": 1.071666705608368, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04313890129948656, + "epoch": 0.21987620096456917, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.941374778747559, + "kd/policy_loss": -3.9117417494455973, + "kd/rkl_advantage_mean": 67.99437721570332, + "kd/rkl_advantage_p95": 90.07292572657268, + "kd/rkl_advantage_std": 36.32029274851084, + "kd/ssd_loss": 0.0, + "learning_rate": 7.801617741996733e-07, + "loss": -15.646967569986979, + "num_tokens": 86282081.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.061332400639852, + "sampling/importance_sampling_ratio/mean": 1.0001226524511972, + "sampling/importance_sampling_ratio/min": 0.8944202343622843, + "sampling/sampling_logp_difference/max": 0.14026769551758964, + "sampling/sampling_logp_difference/mean": 0.0020580797097257647, + "step": 5790, + "step_time": 4.127937762322835, + "student/entropy": 0.04313890129948656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.0688889225323994, + "completions/mean_terminated_length": 1.0688889225323994, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04312944849953056, + "epoch": 0.22101545589184673, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.454360961914062, + "kd/policy_loss": -2.9639599164326986, + "kd/rkl_advantage_mean": 63.062892786661784, + "kd/rkl_advantage_p95": 84.52622042496999, + "kd/rkl_advantage_std": 35.70554228524367, + "kd/ssd_loss": 0.0, + "learning_rate": 7.790225192723958e-07, + "loss": -11.855838012695312, + "num_tokens": 86577001.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.002927307287852, + "sampling/importance_sampling_ratio/mean": 0.9998000760873159, + "sampling/importance_sampling_ratio/min": 0.9106541375319163, + "sampling/sampling_logp_difference/max": 0.0912670565303415, + "sampling/sampling_logp_difference/mean": 0.001653118769172579, + "step": 5820, + "step_time": 4.151564746206471, + "student/entropy": 0.04312944849953056 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.2, + "completions/max_terminated_length": 6.2, + "completions/mean_length": 1.0561111410458883, + "completions/mean_terminated_length": 1.0561111410458883, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.046904389094561336, + "epoch": 0.2221547108191243, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.53268051147461, + "kd/policy_loss": -2.3213027119636536, + "kd/rkl_advantage_mean": 57.30005559921265, + "kd/rkl_advantage_p95": 78.98012237548828, + "kd/rkl_advantage_std": 34.81494497656822, + "kd/ssd_loss": 0.0, + "learning_rate": 7.778832643451183e-07, + "loss": -9.285211181640625, + "num_tokens": 86886595.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0331368764241537, + "sampling/importance_sampling_ratio/mean": 1.0000752230485281, + "sampling/importance_sampling_ratio/min": 0.8982579211393992, + "sampling/sampling_logp_difference/max": 0.1206368259464701, + "sampling/sampling_logp_difference/mean": 0.0018854610001047452, + "step": 5850, + "step_time": 4.1397894227916066, + "student/entropy": 0.046904389094561336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.6, + "completions/max_terminated_length": 6.6, + "completions/mean_length": 1.0866667032241821, + "completions/mean_terminated_length": 1.0866667032241821, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07251719900717338, + "epoch": 0.22329396574640184, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 48.47771453857422, + "kd/policy_loss": -4.338822565476099, + "kd/rkl_advantage_mean": 60.81860105196635, + "kd/rkl_advantage_p95": 78.72613453865051, + "kd/rkl_advantage_std": 29.554045632481575, + "kd/ssd_loss": 0.0, + "learning_rate": 7.767440094178407e-07, + "loss": -17.355291748046874, + "num_tokens": 87193147.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0362922906875611, + "sampling/importance_sampling_ratio/mean": 1.0003947993119557, + "sampling/importance_sampling_ratio/min": 0.9082835992177327, + "sampling/sampling_logp_difference/max": 0.10163516315321128, + "sampling/sampling_logp_difference/mean": 0.0025402775102217373, + "step": 5880, + "step_time": 4.188100990493937, + "student/entropy": 0.07251719900717338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.066666666666666, + "completions/max_terminated_length": 5.066666666666666, + "completions/mean_length": 1.047777811686198, + "completions/mean_terminated_length": 1.047777811686198, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04990175344670812, + "epoch": 0.2244332206736794, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 68.06766510009766, + "kd/policy_loss": -2.1199480990568795, + "kd/rkl_advantage_mean": 45.92759939829509, + "kd/rkl_advantage_p95": 67.12804405689239, + "kd/rkl_advantage_std": 29.483332645893096, + "kd/ssd_loss": 0.0, + "learning_rate": 7.756047544905632e-07, + "loss": -8.479792277018229, + "num_tokens": 87500895.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0476331194241841, + "sampling/importance_sampling_ratio/mean": 1.0001655340194702, + "sampling/importance_sampling_ratio/min": 0.9262278060118357, + "sampling/sampling_logp_difference/max": 0.12381647423220178, + "sampling/sampling_logp_difference/mean": 0.002202614517106364, + "step": 5910, + "step_time": 4.242955929176727, + "student/entropy": 0.04990175344670812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.033333333333333, + "completions/max_terminated_length": 6.033333333333333, + "completions/mean_length": 1.0608333667119345, + "completions/mean_terminated_length": 1.0608333667119345, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05622798629725973, + "epoch": 0.22557247560095697, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 34.015621185302734, + "kd/policy_loss": -1.8729179620742797, + "kd/rkl_advantage_mean": 53.25911018053691, + "kd/rkl_advantage_p95": 73.20334379673004, + "kd/rkl_advantage_std": 32.71277918318908, + "kd/ssd_loss": 0.0, + "learning_rate": 7.744654995632855e-07, + "loss": -7.49167226155599, + "num_tokens": 87806410.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.043274998664856, + "sampling/importance_sampling_ratio/mean": 1.0002846280733744, + "sampling/importance_sampling_ratio/min": 0.9200971643129985, + "sampling/sampling_logp_difference/max": 0.12598953886578482, + "sampling/sampling_logp_difference/mean": 0.002067273793121179, + "step": 5940, + "step_time": 4.193221706342107, + "student/entropy": 0.05622798629725973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.633333333333334, + "completions/max_terminated_length": 6.633333333333334, + "completions/mean_length": 1.0755555987358094, + "completions/mean_terminated_length": 1.0755555987358094, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06268117406095068, + "epoch": 0.22671173052823454, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.149032592773438, + "kd/policy_loss": -3.386896546681722, + "kd/rkl_advantage_mean": 65.82459138234456, + "kd/rkl_advantage_p95": 90.18653922875723, + "kd/rkl_advantage_std": 38.45983721415202, + "kd/ssd_loss": 0.0, + "learning_rate": 7.73326244636008e-07, + "loss": -13.547586059570312, + "num_tokens": 88105482.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0421154220898947, + "sampling/importance_sampling_ratio/mean": 1.0000633259614309, + "sampling/importance_sampling_ratio/min": 0.907740734020869, + "sampling/sampling_logp_difference/max": 0.12490469355446597, + "sampling/sampling_logp_difference/mean": 0.0023456345923477786, + "step": 5970, + "step_time": 4.17124139998341, + "student/entropy": 0.06268117406095068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.2, + "completions/max_terminated_length": 5.2, + "completions/mean_length": 1.0438889106114706, + "completions/mean_terminated_length": 1.0438889106114706, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04960649088025093, + "epoch": 0.2278509854555121, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 72.07746887207031, + "kd/policy_loss": -0.4491792599360148, + "kd/rkl_advantage_mean": 39.289285564422606, + "kd/rkl_advantage_p95": 56.13343474070231, + "kd/rkl_advantage_std": 26.903004437188308, + "kd/ssd_loss": 0.0, + "learning_rate": 7.721869897087304e-07, + "loss": -1.7967166900634766, + "num_tokens": 88413048.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0060701489448547, + "sampling/importance_sampling_ratio/mean": 0.9994105180104573, + "sampling/importance_sampling_ratio/min": 0.9131895542144776, + "sampling/sampling_logp_difference/max": 0.08604397253754238, + "sampling/sampling_logp_difference/mean": 0.0015001348384733623, + "step": 6000, + "step_time": 4.1751805943242895, + "student/entropy": 0.04960649088025093 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.366666666666666, + "completions/max_terminated_length": 6.366666666666666, + "completions/mean_length": 1.0733333627382915, + "completions/mean_terminated_length": 1.0733333627382915, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06100744794433315, + "epoch": 0.22899024038278964, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 42.009525299072266, + "kd/policy_loss": -4.622260775168737, + "kd/rkl_advantage_mean": 69.60772083600362, + "kd/rkl_advantage_p95": 95.47879621187846, + "kd/rkl_advantage_std": 39.0373003060619, + "kd/ssd_loss": 0.0, + "learning_rate": 7.71047734781453e-07, + "loss": -18.489046223958333, + "num_tokens": 88722160.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.1680585861206054, + "sampling/importance_sampling_ratio/mean": 1.0007343590259552, + "sampling/importance_sampling_ratio/min": 0.9155111610889435, + "sampling/sampling_logp_difference/max": 0.18265917464159429, + "sampling/sampling_logp_difference/mean": 0.0034090549558944377, + "step": 6030, + "step_time": 4.255384254168408, + "student/entropy": 0.06100744794433315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.3, + "completions/max_terminated_length": 7.3, + "completions/mean_length": 1.0833333690961202, + "completions/mean_terminated_length": 1.0833333690961202, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06110485459988316, + "epoch": 0.2301294953100672, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 99.89219665527344, + "kd/policy_loss": -4.00811992486318, + "kd/rkl_advantage_mean": 72.19618682861328, + "kd/rkl_advantage_p95": 96.16074221134186, + "kd/rkl_advantage_std": 39.638052538534005, + "kd/ssd_loss": 0.0, + "learning_rate": 7.699084798541753e-07, + "loss": -16.032478841145835, + "num_tokens": 89028420.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.003078250090281, + "sampling/importance_sampling_ratio/mean": 0.999368679523468, + "sampling/importance_sampling_ratio/min": 0.8234932134548824, + "sampling/sampling_logp_difference/max": 0.18264444318289558, + "sampling/sampling_logp_difference/mean": 0.003438991391643261, + "step": 6060, + "step_time": 4.211774904568059, + "student/entropy": 0.06110485459988316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.2, + "completions/max_terminated_length": 6.2, + "completions/mean_length": 1.0702778140703837, + "completions/mean_terminated_length": 1.0702778140703837, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.040652012343828876, + "epoch": 0.23126875023734478, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 57.447811126708984, + "kd/policy_loss": -3.818855845928192, + "kd/rkl_advantage_mean": 59.28879548708598, + "kd/rkl_advantage_p95": 79.4171611626943, + "kd/rkl_advantage_std": 31.438974634806314, + "kd/ssd_loss": 0.0, + "learning_rate": 7.687692249268978e-07, + "loss": -15.275423177083333, + "num_tokens": 89334841.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0022983431816102, + "sampling/importance_sampling_ratio/mean": 0.999122949441274, + "sampling/importance_sampling_ratio/min": 0.8568773031234741, + "sampling/sampling_logp_difference/max": 0.1668266124712924, + "sampling/sampling_logp_difference/mean": 0.0025022130973714712, + "step": 6090, + "step_time": 4.192982976414108, + "student/entropy": 0.040652012343828876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.266666666666667, + "completions/max_terminated_length": 5.266666666666667, + "completions/mean_length": 1.0544444759686789, + "completions/mean_terminated_length": 1.0544444759686789, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.050473350069175166, + "epoch": 0.23240800516462234, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 32.792572021484375, + "kd/policy_loss": -2.2631681164105735, + "kd/rkl_advantage_mean": 45.37689240773519, + "kd/rkl_advantage_p95": 61.54416794776917, + "kd/rkl_advantage_std": 25.991692912578582, + "kd/ssd_loss": 0.0, + "learning_rate": 7.676299699996203e-07, + "loss": -9.05267333984375, + "num_tokens": 89635029.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0340741634368897, + "sampling/importance_sampling_ratio/mean": 1.0003663579622903, + "sampling/importance_sampling_ratio/min": 0.9039239237705866, + "sampling/sampling_logp_difference/max": 0.11434976539264123, + "sampling/sampling_logp_difference/mean": 0.0024424973642453553, + "step": 6120, + "step_time": 4.209282269266745, + "student/entropy": 0.050473350069175166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.833333333333333, + "completions/max_terminated_length": 5.833333333333333, + "completions/mean_length": 1.052500033378601, + "completions/mean_terminated_length": 1.052500033378601, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.037606724988048276, + "epoch": 0.2335472600918999, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.672294616699219, + "kd/policy_loss": -1.7859278202056885, + "kd/rkl_advantage_mean": 51.406966845194496, + "kd/rkl_advantage_p95": 73.09184537728628, + "kd/rkl_advantage_std": 33.01412788728873, + "kd/ssd_loss": 0.0, + "learning_rate": 7.664907150723426e-07, + "loss": -7.1437123616536455, + "num_tokens": 89937682.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0321177879969279, + "sampling/importance_sampling_ratio/mean": 0.9997544467449189, + "sampling/importance_sampling_ratio/min": 0.8826659599939982, + "sampling/sampling_logp_difference/max": 0.13065241862398883, + "sampling/sampling_logp_difference/mean": 0.0021136370623329034, + "step": 6150, + "step_time": 4.184450008999557, + "student/entropy": 0.037606724988048276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.0655555844306945, + "completions/mean_terminated_length": 1.0655555844306945, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04743039136131604, + "epoch": 0.23468651501917745, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.653118133544922, + "kd/policy_loss": -2.855927542845408, + "kd/rkl_advantage_mean": 55.25976111094157, + "kd/rkl_advantage_p95": 73.07089819908143, + "kd/rkl_advantage_std": 30.06801995088657, + "kd/ssd_loss": 0.0, + "learning_rate": 7.653514601450651e-07, + "loss": -11.423709106445312, + "num_tokens": 90249470.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0544631481170654, + "sampling/importance_sampling_ratio/mean": 1.0003834267457326, + "sampling/importance_sampling_ratio/min": 0.9111378828684489, + "sampling/sampling_logp_difference/max": 0.12322648889385164, + "sampling/sampling_logp_difference/mean": 0.0020107527108242113, + "step": 6180, + "step_time": 4.248056435806211, + "student/entropy": 0.04743039136131604 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.058055587609609, + "completions/mean_terminated_length": 1.058055587609609, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04047986331085364, + "epoch": 0.23582576994645502, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.487481117248535, + "kd/policy_loss": -2.274014774958293, + "kd/rkl_advantage_mean": 61.3128667195638, + "kd/rkl_advantage_p95": 84.4521256685257, + "kd/rkl_advantage_std": 37.635977709293364, + "kd/ssd_loss": 0.0, + "learning_rate": 7.642122052177875e-07, + "loss": -9.096059163411459, + "num_tokens": 90549791.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0023425261179606, + "sampling/importance_sampling_ratio/mean": 0.999722683429718, + "sampling/importance_sampling_ratio/min": 0.8979529082775116, + "sampling/sampling_logp_difference/max": 0.11883553395358225, + "sampling/sampling_logp_difference/mean": 0.0017466918914578855, + "step": 6210, + "step_time": 4.214919887840127, + "student/entropy": 0.04047986331085364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.633333333333334, + "completions/max_terminated_length": 6.633333333333334, + "completions/mean_length": 1.066111147403717, + "completions/mean_terminated_length": 1.066111147403717, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0507158185976247, + "epoch": 0.23696502487373258, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.117475509643555, + "kd/policy_loss": -2.6324954549471538, + "kd/rkl_advantage_mean": 61.625254758199056, + "kd/rkl_advantage_p95": 84.13410896460215, + "kd/rkl_advantage_std": 36.487258705496785, + "kd/ssd_loss": 0.0, + "learning_rate": 7.6307295029051e-07, + "loss": -10.529983520507812, + "num_tokens": 90858037.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0313695152600606, + "sampling/importance_sampling_ratio/mean": 1.0003853996594747, + "sampling/importance_sampling_ratio/min": 0.9264123320579529, + "sampling/sampling_logp_difference/max": 0.10915132670973739, + "sampling/sampling_logp_difference/mean": 0.0021029713097959757, + "step": 6240, + "step_time": 4.289577676759412, + "student/entropy": 0.0507158185976247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.333333333333333, + "completions/max_terminated_length": 5.333333333333333, + "completions/mean_length": 1.0500000317891438, + "completions/mean_terminated_length": 1.0500000317891438, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.041663309543703994, + "epoch": 0.23810427980101015, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.81092643737793, + "kd/policy_loss": -1.4342209617296855, + "kd/rkl_advantage_mean": 44.16467138926188, + "kd/rkl_advantage_p95": 61.912240306536354, + "kd/rkl_advantage_std": 27.671006434162457, + "kd/ssd_loss": 0.0, + "learning_rate": 7.619336953632324e-07, + "loss": -5.7368825276692705, + "num_tokens": 91173265.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0061086972554525, + "sampling/importance_sampling_ratio/mean": 0.9997610986232758, + "sampling/importance_sampling_ratio/min": 0.9013271391391754, + "sampling/sampling_logp_difference/max": 0.09574229933011034, + "sampling/sampling_logp_difference/mean": 0.001709340219773973, + "step": 6270, + "step_time": 4.279299347936952, + "student/entropy": 0.041663309543703994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.8, + "completions/max_terminated_length": 6.8, + "completions/mean_length": 1.0686111529668172, + "completions/mean_terminated_length": 1.0686111529668172, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05490114108348886, + "epoch": 0.2392435347282877, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 57.56103515625, + "kd/policy_loss": -4.971296421686808, + "kd/rkl_advantage_mean": 69.42118835449219, + "kd/rkl_advantage_p95": 89.80802504221599, + "kd/rkl_advantage_std": 34.57676875889301, + "kd/ssd_loss": 0.0, + "learning_rate": 7.607944404359549e-07, + "loss": -19.885184733072915, + "num_tokens": 91484912.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0080909093221029, + "sampling/importance_sampling_ratio/mean": 0.9999161064624786, + "sampling/importance_sampling_ratio/min": 0.8810536672671636, + "sampling/sampling_logp_difference/max": 0.13953514105329912, + "sampling/sampling_logp_difference/mean": 0.002353559350982929, + "step": 6300, + "step_time": 4.2183458687776385, + "student/entropy": 0.05490114108348886 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.166666666666667, + "completions/max_terminated_length": 6.166666666666667, + "completions/mean_length": 1.061666695276896, + "completions/mean_terminated_length": 1.061666695276896, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03865435818831126, + "epoch": 0.24038278965556525, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 46.745384216308594, + "kd/policy_loss": -2.5147504687309263, + "kd/rkl_advantage_mean": 54.59612334569295, + "kd/rkl_advantage_p95": 73.46931784947714, + "kd/rkl_advantage_std": 31.05893341700236, + "kd/ssd_loss": 0.0, + "learning_rate": 7.596551855086772e-07, + "loss": -10.059002685546876, + "num_tokens": 91788726.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0029209891955058, + "sampling/importance_sampling_ratio/mean": 0.9992925842603048, + "sampling/importance_sampling_ratio/min": 0.8751725167036056, + "sampling/sampling_logp_difference/max": 0.14147440002610287, + "sampling/sampling_logp_difference/mean": 0.0019001733094531422, + "step": 6330, + "step_time": 4.220617412942617, + "student/entropy": 0.03865435818831126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.0616666992505392, + "completions/mean_terminated_length": 1.0616666992505392, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04934355452035864, + "epoch": 0.24152204458284282, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.990703582763672, + "kd/policy_loss": -2.504650342464447, + "kd/rkl_advantage_mean": 54.29262264569601, + "kd/rkl_advantage_p95": 73.17457446257274, + "kd/rkl_advantage_std": 31.243031639854113, + "kd/ssd_loss": 0.0, + "learning_rate": 7.585159305813997e-07, + "loss": -10.018601481119791, + "num_tokens": 92095044.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0440980195999146, + "sampling/importance_sampling_ratio/mean": 0.9999672551949819, + "sampling/importance_sampling_ratio/min": 0.8740257968505224, + "sampling/sampling_logp_difference/max": 0.1407571566446374, + "sampling/sampling_logp_difference/mean": 0.002336362578595678, + "step": 6360, + "step_time": 4.1583258188058005, + "student/entropy": 0.04934355452035864 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.9, + "completions/max_terminated_length": 6.9, + "completions/mean_length": 1.0819444855054219, + "completions/mean_terminated_length": 1.0819444855054219, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054033813842882715, + "epoch": 0.2426612995101204, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.814739227294922, + "kd/policy_loss": -5.099781415859858, + "kd/rkl_advantage_mean": 65.08513342539469, + "kd/rkl_advantage_p95": 84.48521931966145, + "kd/rkl_advantage_std": 30.826835871736208, + "kd/ssd_loss": 0.0, + "learning_rate": 7.573766756541222e-07, + "loss": -20.399125162760416, + "num_tokens": 92397003.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.015508500734965, + "sampling/importance_sampling_ratio/mean": 0.9998588740825654, + "sampling/importance_sampling_ratio/min": 0.8918997357288997, + "sampling/sampling_logp_difference/max": 0.12893865200070043, + "sampling/sampling_logp_difference/mean": 0.0025298641024467847, + "step": 6390, + "step_time": 4.228777711481476, + "student/entropy": 0.054033813842882715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.4, + "completions/max_terminated_length": 4.4, + "completions/mean_length": 1.0427778085072836, + "completions/mean_terminated_length": 1.0427778085072836, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.050355930160731074, + "epoch": 0.24380055443739795, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 34.037010192871094, + "kd/policy_loss": -1.137835282087326, + "kd/rkl_advantage_mean": 34.37473338445027, + "kd/rkl_advantage_p95": 50.50674078464508, + "kd/rkl_advantage_std": 23.09475687046846, + "kd/ssd_loss": 0.0, + "learning_rate": 7.562374207268446e-07, + "loss": -4.551341756184896, + "num_tokens": 92704549.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.021784774462382, + "sampling/importance_sampling_ratio/mean": 1.0002786974112192, + "sampling/importance_sampling_ratio/min": 0.9252569437026977, + "sampling/sampling_logp_difference/max": 0.12369514439875881, + "sampling/sampling_logp_difference/mean": 0.0020481281622778623, + "step": 6420, + "step_time": 4.280338884536953, + "student/entropy": 0.050355930160731074 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.2, + "completions/max_terminated_length": 7.2, + "completions/mean_length": 1.090000041325887, + "completions/mean_terminated_length": 1.090000041325887, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06953203973049919, + "epoch": 0.2449398093646755, + "eval/gt_acc_batch": 0.002222222338120143, + "frac_reward_zero_std": 1.0, + "grad_norm": 93.4305419921875, + "kd/policy_loss": -4.612080895900727, + "kd/rkl_advantage_mean": 72.80263665517171, + "kd/rkl_advantage_p95": 96.11225959459941, + "kd/rkl_advantage_std": 38.34968491593997, + "kd/ssd_loss": 0.0, + "learning_rate": 7.550981657995671e-07, + "loss": -18.44832560221354, + "num_tokens": 93006873.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.002222222338120143, + "rewards/gt_logging_reward/std": 0.022542615234851838, + "sampling/importance_sampling_ratio/max": 1.0230764150619507, + "sampling/importance_sampling_ratio/mean": 0.9999794999758402, + "sampling/importance_sampling_ratio/min": 0.8607026398181915, + "sampling/sampling_logp_difference/max": 0.17341683165480692, + "sampling/sampling_logp_difference/mean": 0.0028862586652394382, + "step": 6450, + "step_time": 4.15613307885748, + "student/entropy": 0.06953203973049919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.566666666666666, + "completions/max_terminated_length": 5.566666666666666, + "completions/mean_length": 1.0430555820465088, + "completions/mean_terminated_length": 1.0430555820465088, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.041161208009968205, + "epoch": 0.24607906429195306, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.216204643249512, + "kd/policy_loss": -1.2214004913965861, + "kd/rkl_advantage_mean": 47.837733872731526, + "kd/rkl_advantage_p95": 68.02817414601644, + "kd/rkl_advantage_std": 31.611174481113753, + "kd/ssd_loss": 0.0, + "learning_rate": 7.539589108722895e-07, + "loss": -4.885601806640625, + "num_tokens": 93312700.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0226563890775044, + "sampling/importance_sampling_ratio/mean": 1.0002671380837758, + "sampling/importance_sampling_ratio/min": 0.9397123495737711, + "sampling/sampling_logp_difference/max": 0.1013016926124692, + "sampling/sampling_logp_difference/mean": 0.0015055307594593614, + "step": 6480, + "step_time": 4.199856634484604, + "student/entropy": 0.041161208009968205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0788889288902284, + "completions/mean_terminated_length": 1.0788889288902284, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.056301653695603214, + "epoch": 0.24721831921923063, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.460031509399414, + "kd/policy_loss": -3.0979074676831564, + "kd/rkl_advantage_mean": 60.99517092704773, + "kd/rkl_advantage_p95": 84.64721701939901, + "kd/rkl_advantage_std": 35.705473350485164, + "kd/ssd_loss": 0.0, + "learning_rate": 7.52819655945012e-07, + "loss": -12.391629028320313, + "num_tokens": 93614400.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0078911423683166, + "sampling/importance_sampling_ratio/mean": 0.9996132373809814, + "sampling/importance_sampling_ratio/min": 0.87994637688001, + "sampling/sampling_logp_difference/max": 0.15006568043803176, + "sampling/sampling_logp_difference/mean": 0.0028122642155115805, + "step": 6510, + "step_time": 4.196003491012379, + "student/entropy": 0.056301653695603214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.3, + "completions/max_terminated_length": 6.3, + "completions/mean_length": 1.0647222598393757, + "completions/mean_terminated_length": 1.0647222598393757, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05730266310274601, + "epoch": 0.2483575741465082, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.11787986755371, + "kd/policy_loss": -3.827226734161377, + "kd/rkl_advantage_mean": 62.632107957204184, + "kd/rkl_advantage_p95": 84.11985371907552, + "kd/rkl_advantage_std": 34.453179354965684, + "kd/ssd_loss": 0.0, + "learning_rate": 7.516804010177343e-07, + "loss": -15.308909098307291, + "num_tokens": 93929617.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0122807502746582, + "sampling/importance_sampling_ratio/mean": 0.9995977977911631, + "sampling/importance_sampling_ratio/min": 0.8638306905825933, + "sampling/sampling_logp_difference/max": 0.17080641003946465, + "sampling/sampling_logp_difference/mean": 0.002599987422581762, + "step": 6540, + "step_time": 4.259631431176483, + "student/entropy": 0.05730266310274601 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.6, + "completions/max_terminated_length": 6.6, + "completions/mean_length": 1.0561111370722454, + "completions/mean_terminated_length": 1.0561111370722454, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03934135202628871, + "epoch": 0.24949682907378573, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.709142684936523, + "kd/policy_loss": -1.5554069479306538, + "kd/rkl_advantage_mean": 56.45274534225464, + "kd/rkl_advantage_p95": 78.71405127048493, + "kd/rkl_advantage_std": 35.64890066087246, + "kd/ssd_loss": 0.0, + "learning_rate": 7.505411460904568e-07, + "loss": -6.221628824869792, + "num_tokens": 94235259.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0155083497365316, + "sampling/importance_sampling_ratio/mean": 0.9992558856805166, + "sampling/importance_sampling_ratio/min": 0.8622889935970306, + "sampling/sampling_logp_difference/max": 0.1474739717009167, + "sampling/sampling_logp_difference/mean": 0.0020096163789276034, + "step": 6570, + "step_time": 4.125377445268289, + "student/entropy": 0.03934135202628871 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.9, + "completions/max_terminated_length": 5.9, + "completions/mean_length": 1.0563889145851135, + "completions/mean_terminated_length": 1.0563889145851135, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04022333142347634, + "epoch": 0.2506360840010633, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 340.9190368652344, + "kd/policy_loss": -1.4802646478017172, + "kd/rkl_advantage_mean": 48.37698478698731, + "kd/rkl_advantage_p95": 67.53553353150686, + "kd/rkl_advantage_std": 30.47733375529448, + "kd/ssd_loss": 0.0, + "learning_rate": 7.494018911631792e-07, + "loss": -5.92105712890625, + "num_tokens": 94543438.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0835821866989135, + "sampling/importance_sampling_ratio/mean": 1.0000927567481994, + "sampling/importance_sampling_ratio/min": 0.9370060245196025, + "sampling/sampling_logp_difference/max": 0.12085091402598967, + "sampling/sampling_logp_difference/mean": 0.001826017943676561, + "step": 6600, + "step_time": 4.154608258938727, + "student/entropy": 0.04022333142347634 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0005555555845300357, + "completions/max_length": 40.166666666666664, + "completions/max_terminated_length": 6.633333333333334, + "completions/mean_length": 1.3586111625035604, + "completions/mean_terminated_length": 1.0747736096382141, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07268484582503637, + "epoch": 0.25177533892834086, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 78.5831298828125, + "kd/policy_loss": -3.1004257182280224, + "kd/rkl_advantage_mean": 57.07850465774536, + "kd/rkl_advantage_p95": 72.94040393829346, + "kd/rkl_advantage_std": 26.28301692257325, + "kd/ssd_loss": 0.0, + "learning_rate": 7.482626362359017e-07, + "loss": -12.401700846354167, + "num_tokens": 94839241.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.3164969404538474, + "sampling/importance_sampling_ratio/mean": 1.0011525392532348, + "sampling/importance_sampling_ratio/min": 0.8696955482165019, + "sampling/sampling_logp_difference/max": 0.21003073225729169, + "sampling/sampling_logp_difference/mean": 0.005144380679121241, + "step": 6630, + "step_time": 4.3499875494473, + "student/entropy": 0.07268484582503637 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.366666666666667, + "completions/max_terminated_length": 8.366666666666667, + "completions/mean_length": 1.103611167271932, + "completions/mean_terminated_length": 1.103611167271932, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07711503012105822, + "epoch": 0.25291459385561843, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.002523422241211, + "kd/policy_loss": -7.295801891883214, + "kd/rkl_advantage_mean": 90.67498343785604, + "kd/rkl_advantage_p95": 118.74867005348206, + "kd/rkl_advantage_std": 44.1763574898243, + "kd/ssd_loss": 0.0, + "learning_rate": 7.471233813086242e-07, + "loss": -29.183209228515626, + "num_tokens": 95142366.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.169377052783966, + "sampling/importance_sampling_ratio/mean": 0.9999233345190685, + "sampling/importance_sampling_ratio/min": 0.8576018770535787, + "sampling/sampling_logp_difference/max": 0.23506320860857766, + "sampling/sampling_logp_difference/mean": 0.005066810073913075, + "step": 6660, + "step_time": 4.209633784900264, + "student/entropy": 0.07711503012105822 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.766666666666667, + "completions/max_terminated_length": 5.766666666666667, + "completions/mean_length": 1.0419444719950357, + "completions/mean_terminated_length": 1.0419444719950357, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03825047379359603, + "epoch": 0.254053848782896, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.48650360107422, + "kd/policy_loss": -1.1273565431435904, + "kd/rkl_advantage_mean": 50.28334019978841, + "kd/rkl_advantage_p95": 72.985378130277, + "kd/rkl_advantage_std": 34.44596322675546, + "kd/ssd_loss": 0.0, + "learning_rate": 7.459841263813466e-07, + "loss": -4.509425862630208, + "num_tokens": 95444317.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0139163970947265, + "sampling/importance_sampling_ratio/mean": 1.000043261051178, + "sampling/importance_sampling_ratio/min": 0.928421417872111, + "sampling/sampling_logp_difference/max": 0.08822365058586001, + "sampling/sampling_logp_difference/mean": 0.0013533474402114129, + "step": 6690, + "step_time": 4.228452419272314, + "student/entropy": 0.03825047379359603 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.433333333333334, + "completions/max_terminated_length": 5.433333333333334, + "completions/mean_length": 1.060277811686198, + "completions/mean_terminated_length": 1.060277811686198, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04412649863710006, + "epoch": 0.25519310371017356, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 46.838497161865234, + "kd/policy_loss": -1.6360430757204691, + "kd/rkl_advantage_mean": 45.8621862411499, + "kd/rkl_advantage_p95": 67.18169101874034, + "kd/rkl_advantage_std": 30.335634045302868, + "kd/ssd_loss": 0.0, + "learning_rate": 7.44844871454069e-07, + "loss": -6.544170633951823, + "num_tokens": 95748174.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0648406505584718, + "sampling/importance_sampling_ratio/mean": 1.0000552912553151, + "sampling/importance_sampling_ratio/min": 0.907900216182073, + "sampling/sampling_logp_difference/max": 0.11899006646126509, + "sampling/sampling_logp_difference/mean": 0.0021169980813283474, + "step": 6720, + "step_time": 4.166885596253754, + "student/entropy": 0.04412649863710006 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.833333333333333, + "completions/max_terminated_length": 6.833333333333333, + "completions/mean_length": 1.0655555923779805, + "completions/mean_terminated_length": 1.0655555923779805, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.053237029785911244, + "epoch": 0.25633235863745113, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 26.86773109436035, + "kd/policy_loss": -3.0491541822751365, + "kd/rkl_advantage_mean": 65.93839743932088, + "kd/rkl_advantage_p95": 90.15830256938935, + "kd/rkl_advantage_std": 38.853148440519966, + "kd/ssd_loss": 0.0, + "learning_rate": 7.437056165267914e-07, + "loss": -12.196617635091146, + "num_tokens": 96055074.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0139097134272257, + "sampling/importance_sampling_ratio/mean": 0.9998467882474263, + "sampling/importance_sampling_ratio/min": 0.8913848082224528, + "sampling/sampling_logp_difference/max": 0.12462219536925355, + "sampling/sampling_logp_difference/mean": 0.0022618908415703723, + "step": 6750, + "step_time": 4.193462803556273, + "student/entropy": 0.053237029785911244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.7, + "completions/max_terminated_length": 6.7, + "completions/mean_length": 1.0791667064030965, + "completions/mean_terminated_length": 1.0791667064030965, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04126878959747652, + "epoch": 0.25747161356472864, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 75.43267822265625, + "kd/policy_loss": -3.9863587816556296, + "kd/rkl_advantage_mean": 66.55280437469483, + "kd/rkl_advantage_p95": 90.35501373608908, + "kd/rkl_advantage_std": 36.80760987401008, + "kd/ssd_loss": 0.0, + "learning_rate": 7.425663615995139e-07, + "loss": -15.945437622070312, + "num_tokens": 96355559.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.028844428062439, + "sampling/importance_sampling_ratio/mean": 0.9994431893030803, + "sampling/importance_sampling_ratio/min": 0.8728853762149811, + "sampling/sampling_logp_difference/max": 0.16529492327633002, + "sampling/sampling_logp_difference/mean": 0.0024274222028907388, + "step": 6780, + "step_time": 4.206636120891199, + "student/entropy": 0.04126878959747652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.433333333333334, + "completions/max_terminated_length": 5.433333333333334, + "completions/mean_length": 1.0561111370722454, + "completions/mean_terminated_length": 1.0561111370722454, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04556716888522108, + "epoch": 0.2586108684920062, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 38.855613708496094, + "kd/policy_loss": -1.4969796140988667, + "kd/rkl_advantage_mean": 44.91370646158854, + "kd/rkl_advantage_p95": 62.34887269337972, + "kd/rkl_advantage_std": 27.87478392223517, + "kd/ssd_loss": 0.0, + "learning_rate": 7.414271066722363e-07, + "loss": -5.987916564941406, + "num_tokens": 96676097.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.032266116142273, + "sampling/importance_sampling_ratio/mean": 1.0003211736679076, + "sampling/importance_sampling_ratio/min": 0.9179418563842774, + "sampling/sampling_logp_difference/max": 0.10432127066887915, + "sampling/sampling_logp_difference/mean": 0.002053998583384479, + "step": 6810, + "step_time": 4.218719320499804, + "student/entropy": 0.04556716888522108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.833333333333333, + "completions/max_terminated_length": 6.833333333333333, + "completions/mean_length": 1.0741667071978251, + "completions/mean_terminated_length": 1.0741667071978251, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05503547675907612, + "epoch": 0.2597501234192838, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 32.86094284057617, + "kd/policy_loss": -3.2587959706783294, + "kd/rkl_advantage_mean": 64.84587065378825, + "kd/rkl_advantage_p95": 89.76037259101868, + "kd/rkl_advantage_std": 38.0845366636912, + "kd/ssd_loss": 0.0, + "learning_rate": 7.402878517449588e-07, + "loss": -13.0351806640625, + "num_tokens": 96982468.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0315672079722087, + "sampling/importance_sampling_ratio/mean": 1.0002035240332285, + "sampling/importance_sampling_ratio/min": 0.8985718826452891, + "sampling/sampling_logp_difference/max": 0.12727264982337752, + "sampling/sampling_logp_difference/mean": 0.002499000012176111, + "step": 6840, + "step_time": 4.1795359462771255, + "student/entropy": 0.05503547675907612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.7, + "completions/max_terminated_length": 5.7, + "completions/mean_length": 1.0441666960716247, + "completions/mean_terminated_length": 1.0441666960716247, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.034702453187977274, + "epoch": 0.26088937834656134, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.522112846374512, + "kd/policy_loss": -1.44053422609965, + "kd/rkl_advantage_mean": 50.90215430259705, + "kd/rkl_advantage_p95": 73.12285346190134, + "kd/rkl_advantage_std": 33.97530072728793, + "kd/ssd_loss": 0.0, + "learning_rate": 7.391485968176812e-07, + "loss": -5.762138875325521, + "num_tokens": 97284619.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0396368463834127, + "sampling/importance_sampling_ratio/mean": 1.0004482726256052, + "sampling/importance_sampling_ratio/min": 0.9444578190644582, + "sampling/sampling_logp_difference/max": 0.10630238411637644, + "sampling/sampling_logp_difference/mean": 0.0016727461411695306, + "step": 6870, + "step_time": 4.250785907831354, + "student/entropy": 0.034702453187977274 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.8, + "completions/max_terminated_length": 5.8, + "completions/mean_length": 1.0588889201482137, + "completions/mean_terminated_length": 1.0588889201482137, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.040150663986181216, + "epoch": 0.2620286332738389, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 36.73229217529297, + "kd/policy_loss": -2.5180691957473753, + "kd/rkl_advantage_mean": 53.89660639762879, + "kd/rkl_advantage_p95": 73.2372376759847, + "kd/rkl_advantage_std": 31.67770516773065, + "kd/ssd_loss": 0.0, + "learning_rate": 7.380093418904036e-07, + "loss": -10.07227783203125, + "num_tokens": 97596623.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0026670535405477, + "sampling/importance_sampling_ratio/mean": 0.9996376474698384, + "sampling/importance_sampling_ratio/min": 0.8918871720631917, + "sampling/sampling_logp_difference/max": 0.11443424510459105, + "sampling/sampling_logp_difference/mean": 0.001847944757901132, + "step": 6900, + "step_time": 4.237888312215606, + "student/entropy": 0.040150663986181216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0655555923779805, + "completions/mean_terminated_length": 1.0655555923779805, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04679239597171545, + "epoch": 0.2631678882011165, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.92816162109375, + "kd/policy_loss": -3.552244122823079, + "kd/rkl_advantage_mean": 62.83442727724711, + "kd/rkl_advantage_p95": 84.4252895196279, + "kd/rkl_advantage_std": 34.96130018432935, + "kd/ssd_loss": 0.0, + "learning_rate": 7.368700869631261e-07, + "loss": -14.208978271484375, + "num_tokens": 97910379.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0161795179049173, + "sampling/importance_sampling_ratio/mean": 0.9998187005519867, + "sampling/importance_sampling_ratio/min": 0.9019032180309295, + "sampling/sampling_logp_difference/max": 0.10895254267379642, + "sampling/sampling_logp_difference/mean": 0.00214147880130137, + "step": 6930, + "step_time": 4.313030561851337, + "student/entropy": 0.04679239597171545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.533333333333333, + "completions/max_terminated_length": 5.533333333333333, + "completions/mean_length": 1.0544444719950359, + "completions/mean_terminated_length": 1.0544444719950359, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0436441894620657, + "epoch": 0.26430714312839404, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 122.35527801513672, + "kd/policy_loss": -2.131202721595764, + "kd/rkl_advantage_mean": 49.00515286127726, + "kd/rkl_advantage_p95": 67.6899698416392, + "kd/rkl_advantage_std": 29.46681151390076, + "kd/ssd_loss": 0.0, + "learning_rate": 7.357308320358485e-07, + "loss": -8.524812825520833, + "num_tokens": 98215071.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0113566040992736, + "sampling/importance_sampling_ratio/mean": 1.0001620431741078, + "sampling/importance_sampling_ratio/min": 0.9365282734235127, + "sampling/sampling_logp_difference/max": 0.0860986457982411, + "sampling/sampling_logp_difference/mean": 0.0017223285462629671, + "step": 6960, + "step_time": 4.149658608161068, + "student/entropy": 0.0436441894620657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.2, + "completions/max_terminated_length": 6.2, + "completions/mean_length": 1.071666697661082, + "completions/mean_terminated_length": 1.071666697661082, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05268490634237726, + "epoch": 0.2654463980556716, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.105072021484375, + "kd/policy_loss": -3.160026148955027, + "kd/rkl_advantage_mean": 52.19787913958232, + "kd/rkl_advantage_p95": 67.98828030427298, + "kd/rkl_advantage_std": 26.568147432307402, + "kd/ssd_loss": 0.0, + "learning_rate": 7.34591577108571e-07, + "loss": -12.640104166666667, + "num_tokens": 98532425.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0621735135714212, + "sampling/importance_sampling_ratio/mean": 1.0004116952419282, + "sampling/importance_sampling_ratio/min": 0.9192578772703807, + "sampling/sampling_logp_difference/max": 0.12886591603358585, + "sampling/sampling_logp_difference/mean": 0.002703264099545777, + "step": 6990, + "step_time": 4.226590462475239, + "student/entropy": 0.05268490634237726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0805555939674378, + "completions/mean_terminated_length": 1.0805555939674378, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.045760315470397474, + "epoch": 0.2665856529829492, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.412075996398926, + "kd/policy_loss": -3.4148519774278006, + "kd/rkl_advantage_mean": 62.47609728177388, + "kd/rkl_advantage_p95": 84.79642318089803, + "kd/rkl_advantage_std": 34.64435287018617, + "kd/ssd_loss": 0.0, + "learning_rate": 7.334523221812934e-07, + "loss": -13.659407552083334, + "num_tokens": 98833555.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0287218809127807, + "sampling/importance_sampling_ratio/mean": 1.000077881415685, + "sampling/importance_sampling_ratio/min": 0.9147246936957042, + "sampling/sampling_logp_difference/max": 0.13090068898163737, + "sampling/sampling_logp_difference/mean": 0.0023481941723730417, + "step": 7020, + "step_time": 4.155057375105874, + "student/entropy": 0.045760315470397474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.7, + "completions/max_terminated_length": 5.7, + "completions/mean_length": 1.0530555884043376, + "completions/mean_terminated_length": 1.0530555884043376, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03846237822435796, + "epoch": 0.26772490791022674, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.634300231933594, + "kd/policy_loss": -2.3295734306176503, + "kd/rkl_advantage_mean": 51.43620835940043, + "kd/rkl_advantage_p95": 72.86833504835765, + "kd/rkl_advantage_std": 32.15803971389929, + "kd/ssd_loss": 0.0, + "learning_rate": 7.323130672540159e-07, + "loss": -9.318294270833333, + "num_tokens": 99138994.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0211313605308532, + "sampling/importance_sampling_ratio/mean": 1.0000448366006216, + "sampling/importance_sampling_ratio/min": 0.9181882739067078, + "sampling/sampling_logp_difference/max": 0.11051033114393552, + "sampling/sampling_logp_difference/mean": 0.0018217960003918657, + "step": 7050, + "step_time": 4.158740484586451, + "student/entropy": 0.03846237822435796 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.833333333333333, + "completions/max_terminated_length": 5.833333333333333, + "completions/mean_length": 1.0677778164545695, + "completions/mean_terminated_length": 1.0677778164545695, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05908559852590164, + "epoch": 0.26886416283750425, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.062847137451172, + "kd/policy_loss": -3.023613687356313, + "kd/rkl_advantage_mean": 56.56094245910644, + "kd/rkl_advantage_p95": 79.01613891124725, + "kd/rkl_advantage_std": 33.41628632843494, + "kd/ssd_loss": 0.0, + "learning_rate": 7.311738123267383e-07, + "loss": -12.094452921549479, + "num_tokens": 99444750.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0381253997484843, + "sampling/importance_sampling_ratio/mean": 1.0006008823712667, + "sampling/importance_sampling_ratio/min": 0.9285407741864522, + "sampling/sampling_logp_difference/max": 0.10402873931452632, + "sampling/sampling_logp_difference/mean": 0.002282324737946813, + "step": 7080, + "step_time": 4.258662170513222, + "student/entropy": 0.05908559852590164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.733333333333333, + "completions/max_terminated_length": 5.733333333333333, + "completions/mean_length": 1.051111137866974, + "completions/mean_terminated_length": 1.051111137866974, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03188577241574724, + "epoch": 0.2700034177647818, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.25019359588623, + "kd/policy_loss": -1.90344447294871, + "kd/rkl_advantage_mean": 46.197894954681395, + "kd/rkl_advantage_p95": 62.35989332199097, + "kd/rkl_advantage_std": 26.63001953959465, + "kd/ssd_loss": 0.0, + "learning_rate": 7.300345573994607e-07, + "loss": -7.613778686523437, + "num_tokens": 99760590.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.005137276649475, + "sampling/importance_sampling_ratio/mean": 0.9996768216292063, + "sampling/importance_sampling_ratio/min": 0.9280817528565725, + "sampling/sampling_logp_difference/max": 0.07614707353835305, + "sampling/sampling_logp_difference/mean": 0.0012524772959295661, + "step": 7110, + "step_time": 4.243415654310956, + "student/entropy": 0.03188577241574724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.8, + "completions/max_terminated_length": 6.8, + "completions/mean_length": 1.0675000349680583, + "completions/mean_terminated_length": 1.0675000349680583, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04796635983511806, + "epoch": 0.2711426726920594, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.343313217163086, + "kd/policy_loss": -3.29514594078064, + "kd/rkl_advantage_mean": 63.14910449981689, + "kd/rkl_advantage_p95": 84.40088636080424, + "kd/rkl_advantage_std": 35.09331017980973, + "kd/ssd_loss": 0.0, + "learning_rate": 7.288953024721831e-07, + "loss": -13.18058573404948, + "num_tokens": 100070697.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0141510764757793, + "sampling/importance_sampling_ratio/mean": 0.9998801132043202, + "sampling/importance_sampling_ratio/min": 0.9108982483545939, + "sampling/sampling_logp_difference/max": 0.09886643995220462, + "sampling/sampling_logp_difference/mean": 0.001838831133985271, + "step": 7140, + "step_time": 4.205869795552766, + "student/entropy": 0.04796635983511806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.666666666666667, + "completions/max_terminated_length": 5.666666666666667, + "completions/mean_length": 1.046666693687439, + "completions/mean_terminated_length": 1.046666693687439, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.040723152039572594, + "epoch": 0.27228192761933695, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.310958862304688, + "kd/policy_loss": -0.8240795175234477, + "kd/rkl_advantage_mean": 47.21908949216207, + "kd/rkl_advantage_p95": 67.60127464930217, + "kd/rkl_advantage_std": 32.220261778434114, + "kd/ssd_loss": 0.0, + "learning_rate": 7.277560475449056e-07, + "loss": -3.296317545572917, + "num_tokens": 100383969.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0045218825340272, + "sampling/importance_sampling_ratio/mean": 0.9993293404579162, + "sampling/importance_sampling_ratio/min": 0.885249400138855, + "sampling/sampling_logp_difference/max": 0.13106151145572464, + "sampling/sampling_logp_difference/mean": 0.0017298986659928536, + "step": 7170, + "step_time": 4.339820748113562, + "student/entropy": 0.040723152039572594 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.166666666666667, + "completions/max_terminated_length": 6.166666666666667, + "completions/mean_length": 1.0691667040189108, + "completions/mean_terminated_length": 1.0691667040189108, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05040450080608328, + "epoch": 0.2734211825466145, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.328372955322266, + "kd/policy_loss": -3.907518188158671, + "kd/rkl_advantage_mean": 57.2282306989034, + "kd/rkl_advantage_p95": 73.6893541653951, + "kd/rkl_advantage_std": 28.04601647257805, + "kd/ssd_loss": 0.0, + "learning_rate": 7.266167926176281e-07, + "loss": -15.630072021484375, + "num_tokens": 100689586.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0382921934127807, + "sampling/importance_sampling_ratio/mean": 0.9999066213766734, + "sampling/importance_sampling_ratio/min": 0.9087083280086518, + "sampling/sampling_logp_difference/max": 0.13503623752233881, + "sampling/sampling_logp_difference/mean": 0.0022999325990288827, + "step": 7200, + "step_time": 4.274129767665484, + "student/entropy": 0.05040450080608328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.9, + "completions/max_terminated_length": 6.9, + "completions/mean_length": 1.0788889288902284, + "completions/mean_terminated_length": 1.0788889288902284, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04759462804843982, + "epoch": 0.2745604374738921, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.97333335876465, + "kd/policy_loss": -5.166953140497208, + "kd/rkl_advantage_mean": 69.4232531706492, + "kd/rkl_advantage_p95": 90.58705548445384, + "kd/rkl_advantage_std": 34.35771046777566, + "kd/ssd_loss": 0.0, + "learning_rate": 7.254775376903505e-07, + "loss": -20.66781209309896, + "num_tokens": 101013486.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0981041193008423, + "sampling/importance_sampling_ratio/mean": 1.0004054168860117, + "sampling/importance_sampling_ratio/min": 0.9132810413837433, + "sampling/sampling_logp_difference/max": 0.1600254628341645, + "sampling/sampling_logp_difference/mean": 0.0023927812774976096, + "step": 7230, + "step_time": 4.356244254806855, + "student/entropy": 0.04759462804843982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.0625000278155008, + "completions/mean_terminated_length": 1.0625000278155008, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04841931142533819, + "epoch": 0.27569969240116965, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.265077590942383, + "kd/policy_loss": -3.191357215245565, + "kd/rkl_advantage_mean": 55.628067223231, + "kd/rkl_advantage_p95": 73.47074734369913, + "kd/rkl_advantage_std": 29.485874419411022, + "kd/ssd_loss": 0.0, + "learning_rate": 7.24338282763073e-07, + "loss": -12.765427652994791, + "num_tokens": 101315847.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0050405859947205, + "sampling/importance_sampling_ratio/mean": 0.9996108174324035, + "sampling/importance_sampling_ratio/min": 0.884422759215037, + "sampling/sampling_logp_difference/max": 0.11586813301158448, + "sampling/sampling_logp_difference/mean": 0.002020783144204567, + "step": 7260, + "step_time": 4.214669422976052, + "student/entropy": 0.04841931142533819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.766666666666667, + "completions/max_terminated_length": 5.766666666666667, + "completions/mean_length": 1.0594444791475932, + "completions/mean_terminated_length": 1.0594444791475932, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05676092539603512, + "epoch": 0.2768389473284472, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.413326263427734, + "kd/policy_loss": -2.503141689300537, + "kd/rkl_advantage_mean": 50.48732248942057, + "kd/rkl_advantage_p95": 67.8374849875768, + "kd/rkl_advantage_std": 28.37336620489756, + "kd/ssd_loss": 0.0, + "learning_rate": 7.231990278357953e-07, + "loss": -10.012567138671875, + "num_tokens": 101621229.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0333418488502502, + "sampling/importance_sampling_ratio/mean": 1.0003665586312611, + "sampling/importance_sampling_ratio/min": 0.9184083839257559, + "sampling/sampling_logp_difference/max": 0.09893469228409231, + "sampling/sampling_logp_difference/mean": 0.0019088564265985041, + "step": 7290, + "step_time": 4.24259037602848, + "student/entropy": 0.05676092539603512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0583333611488341, + "completions/mean_terminated_length": 1.0583333611488341, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04231608686968684, + "epoch": 0.2779782022557248, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.810059547424316, + "kd/policy_loss": -2.503264621893565, + "kd/rkl_advantage_mean": 53.63413999875387, + "kd/rkl_advantage_p95": 73.18590001265208, + "kd/rkl_advantage_std": 31.1744007324179, + "kd/ssd_loss": 0.0, + "learning_rate": 7.220597729085178e-07, + "loss": -10.013059488932292, + "num_tokens": 101935015.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.022873302300771, + "sampling/importance_sampling_ratio/mean": 0.9998563508192698, + "sampling/importance_sampling_ratio/min": 0.9252212931712468, + "sampling/sampling_logp_difference/max": 0.09280167225127418, + "sampling/sampling_logp_difference/mean": 0.0017662319442024454, + "step": 7320, + "step_time": 4.3551615443119465, + "student/entropy": 0.04231608686968684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.733333333333333, + "completions/max_terminated_length": 6.733333333333333, + "completions/mean_length": 1.073888929684957, + "completions/mean_terminated_length": 1.073888929684957, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04991802331060171, + "epoch": 0.2791174571830023, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.272061347961426, + "kd/policy_loss": -3.827504857381185, + "kd/rkl_advantage_mean": 66.57599720954894, + "kd/rkl_advantage_p95": 89.97275983492533, + "kd/rkl_advantage_std": 37.03917325039705, + "kd/ssd_loss": 0.0, + "learning_rate": 7.209205179812402e-07, + "loss": -15.310017903645834, + "num_tokens": 102240689.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0151294191678366, + "sampling/importance_sampling_ratio/mean": 0.9996023774147034, + "sampling/importance_sampling_ratio/min": 0.8736132919788361, + "sampling/sampling_logp_difference/max": 0.1451751540104548, + "sampling/sampling_logp_difference/mean": 0.0026006167851543673, + "step": 7350, + "step_time": 4.175210376440858, + "student/entropy": 0.04991802331060171 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.4, + "completions/max_terminated_length": 5.4, + "completions/mean_length": 1.0577778061230978, + "completions/mean_terminated_length": 1.0577778061230978, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04430676200427115, + "epoch": 0.28025671211027986, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 79.92292785644531, + "kd/policy_loss": -2.079970633983612, + "kd/rkl_advantage_mean": 42.283765411376955, + "kd/rkl_advantage_p95": 56.329790655771895, + "kd/rkl_advantage_std": 23.025529050827025, + "kd/ssd_loss": 0.0, + "learning_rate": 7.197812630539626e-07, + "loss": -8.31988271077474, + "num_tokens": 102544057.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0341959595680237, + "sampling/importance_sampling_ratio/mean": 0.9997346222400665, + "sampling/importance_sampling_ratio/min": 0.9021895845731099, + "sampling/sampling_logp_difference/max": 0.13502568951807917, + "sampling/sampling_logp_difference/mean": 0.002216991109889932, + "step": 7380, + "step_time": 4.156548597357081, + "student/entropy": 0.04430676200427115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.366666666666666, + "completions/max_terminated_length": 7.366666666666666, + "completions/mean_length": 1.0794444799423217, + "completions/mean_terminated_length": 1.0794444799423217, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05384270803381999, + "epoch": 0.2813959670375574, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 54.308170318603516, + "kd/policy_loss": -4.0156831463178, + "kd/rkl_advantage_mean": 68.33174227078756, + "kd/rkl_advantage_p95": 90.39568022092183, + "kd/rkl_advantage_std": 36.025144352018835, + "kd/ssd_loss": 0.0, + "learning_rate": 7.186420081266851e-07, + "loss": -16.06273193359375, + "num_tokens": 102850703.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0311777551968893, + "sampling/importance_sampling_ratio/mean": 0.9996054033438365, + "sampling/importance_sampling_ratio/min": 0.894509736696879, + "sampling/sampling_logp_difference/max": 0.12767322727789482, + "sampling/sampling_logp_difference/mean": 0.002207245914420734, + "step": 7410, + "step_time": 4.245261039875913, + "student/entropy": 0.05384270803381999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.066666666666666, + "completions/max_terminated_length": 6.066666666666666, + "completions/mean_length": 1.0522222518920898, + "completions/mean_terminated_length": 1.0522222518920898, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03769344215591749, + "epoch": 0.282535221964835, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.446826934814453, + "kd/policy_loss": -1.796246631940206, + "kd/rkl_advantage_mean": 51.69400122960408, + "kd/rkl_advantage_p95": 72.85626897811889, + "kd/rkl_advantage_std": 32.52872983912627, + "kd/ssd_loss": 0.0, + "learning_rate": 7.175027531994076e-07, + "loss": -7.184986877441406, + "num_tokens": 103160803.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0472587545712788, + "sampling/importance_sampling_ratio/mean": 0.9999058385690053, + "sampling/importance_sampling_ratio/min": 0.9111975729465485, + "sampling/sampling_logp_difference/max": 0.160151478741318, + "sampling/sampling_logp_difference/mean": 0.0021746561789768746, + "step": 7440, + "step_time": 4.2109339156032854, + "student/entropy": 0.03769344215591749 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.5, + "completions/max_terminated_length": 6.5, + "completions/mean_length": 1.075000027815501, + "completions/mean_terminated_length": 1.075000027815501, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05715933283790946, + "epoch": 0.28367447689211256, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 29.490304946899414, + "kd/policy_loss": -2.789228653907776, + "kd/rkl_advantage_mean": 55.16221137046814, + "kd/rkl_advantage_p95": 73.32393546104431, + "kd/rkl_advantage_std": 30.112378109494845, + "kd/ssd_loss": 0.0, + "learning_rate": 7.1636349827213e-07, + "loss": -11.156917317708333, + "num_tokens": 103470985.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0391608754793802, + "sampling/importance_sampling_ratio/mean": 0.9999450147151947, + "sampling/importance_sampling_ratio/min": 0.9111386408408483, + "sampling/sampling_logp_difference/max": 0.11920477516638736, + "sampling/sampling_logp_difference/mean": 0.0021208752101908126, + "step": 7470, + "step_time": 4.259058026992716, + "student/entropy": 0.05715933283790946 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.793103448275862, + "completions/max_terminated_length": 7.793103448275862, + "completions/mean_length": 1.0778735958296677, + "completions/mean_terminated_length": 1.0778735958296677, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06381526916962245, + "epoch": 0.2848137318193901, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.899423599243164, + "kd/policy_loss": -4.047297777800725, + "kd/rkl_advantage_mean": 71.86498286806304, + "kd/rkl_advantage_p95": 93.90799233831208, + "kd/rkl_advantage_std": 37.43731732101276, + "kd/ssd_loss": 0.0, + "learning_rate": 7.152242433448524e-07, + "loss": -15.649552408854166, + "num_tokens": 103758464.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.024101709497386, + "sampling/importance_sampling_ratio/mean": 0.9993926451123994, + "sampling/importance_sampling_ratio/min": 0.8725594744600099, + "sampling/sampling_logp_difference/max": 0.1591628260011303, + "sampling/sampling_logp_difference/mean": 0.0024629119384214923, + "step": 7500, + "step_time": 4.12382629486965, + "student/entropy": 0.06381526916962245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.466666666666667, + "completions/max_terminated_length": 5.466666666666667, + "completions/mean_length": 1.0650000294049582, + "completions/mean_terminated_length": 1.0650000294049582, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05362552848334114, + "epoch": 0.2859529867466677, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 88.41254425048828, + "kd/policy_loss": -1.2761667490005493, + "kd/rkl_advantage_mean": 40.84257345199585, + "kd/rkl_advantage_p95": 56.06154545148214, + "kd/rkl_advantage_std": 24.69555477052927, + "kd/ssd_loss": 0.0, + "learning_rate": 7.140849884175749e-07, + "loss": -5.104668172200521, + "num_tokens": 104059106.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0085193157196044, + "sampling/importance_sampling_ratio/mean": 0.9994846920172373, + "sampling/importance_sampling_ratio/min": 0.9003282845020294, + "sampling/sampling_logp_difference/max": 0.0741852052975446, + "sampling/sampling_logp_difference/mean": 0.0017280648569188391, + "step": 7530, + "step_time": 4.2402986734756265, + "student/entropy": 0.05362552848334114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.4, + "completions/max_terminated_length": 7.4, + "completions/mean_length": 1.080833375453949, + "completions/mean_terminated_length": 1.080833375453949, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05822457863638798, + "epoch": 0.28709224167394526, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.20414161682129, + "kd/policy_loss": -5.115128529071808, + "kd/rkl_advantage_mean": 71.2141029993693, + "kd/rkl_advantage_p95": 90.71889878908793, + "kd/rkl_advantage_std": 33.889059547583265, + "kd/ssd_loss": 0.0, + "learning_rate": 7.129457334902973e-07, + "loss": -20.460514322916666, + "num_tokens": 104368213.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.040657114982605, + "sampling/importance_sampling_ratio/mean": 0.9998092492421468, + "sampling/importance_sampling_ratio/min": 0.8836849908034007, + "sampling/sampling_logp_difference/max": 0.12993133934214712, + "sampling/sampling_logp_difference/mean": 0.0021324759174603967, + "step": 7560, + "step_time": 4.406674345877643, + "student/entropy": 0.05822457863638798 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.6, + "completions/max_terminated_length": 6.6, + "completions/mean_length": 1.0627778053283692, + "completions/mean_terminated_length": 1.0627778053283692, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04799731262028217, + "epoch": 0.2882314966012228, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.76934242248535, + "kd/policy_loss": -2.592785402139028, + "kd/rkl_advantage_mean": 55.26148505210877, + "kd/rkl_advantage_p95": 73.7100396156311, + "kd/rkl_advantage_std": 30.895066609978677, + "kd/ssd_loss": 0.0, + "learning_rate": 7.118064785630198e-07, + "loss": -10.371140543619791, + "num_tokens": 104676711.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.1116754094759622, + "sampling/importance_sampling_ratio/mean": 1.0005326410134634, + "sampling/importance_sampling_ratio/min": 0.9164291779200237, + "sampling/sampling_logp_difference/max": 0.1251769663921247, + "sampling/sampling_logp_difference/mean": 0.0021426516633558396, + "step": 7590, + "step_time": 4.399678580761732, + "student/entropy": 0.04799731262028217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 23.1, + "completions/max_terminated_length": 6.066666666666666, + "completions/mean_length": 1.1930555939674377, + "completions/mean_terminated_length": 1.051111137866974, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.044782804263134794, + "epoch": 0.28937075152850034, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.012077331542969, + "kd/policy_loss": -1.4123799512783686, + "kd/rkl_advantage_mean": 53.81074854532878, + "kd/rkl_advantage_p95": 73.21784706910451, + "kd/rkl_advantage_std": 31.17184480180343, + "kd/ssd_loss": 0.0, + "learning_rate": 7.106672236357422e-07, + "loss": -5.649518839518229, + "num_tokens": 104972974.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0180897196133931, + "sampling/importance_sampling_ratio/mean": 0.9997439006964366, + "sampling/importance_sampling_ratio/min": 0.9156634231408437, + "sampling/sampling_logp_difference/max": 0.12242698945725958, + "sampling/sampling_logp_difference/mean": 0.0018300648322716977, + "step": 7620, + "step_time": 4.4533910058555195, + "student/entropy": 0.044782804263134794 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.076388923327128, + "completions/mean_terminated_length": 1.076388923327128, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05658193497608106, + "epoch": 0.2905100064557779, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.321449279785156, + "kd/policy_loss": -3.4752560536066692, + "kd/rkl_advantage_mean": 56.392937088012694, + "kd/rkl_advantage_p95": 73.49644033908844, + "kd/rkl_advantage_std": 29.04337693452835, + "kd/ssd_loss": 0.0, + "learning_rate": 7.095279687084647e-07, + "loss": -13.901021321614584, + "num_tokens": 105288561.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0271222829818725, + "sampling/importance_sampling_ratio/mean": 0.9998455166816711, + "sampling/importance_sampling_ratio/min": 0.9176694571971893, + "sampling/sampling_logp_difference/max": 0.10228924630209804, + "sampling/sampling_logp_difference/mean": 0.0019241100633128857, + "step": 7650, + "step_time": 4.441868524051582, + "student/entropy": 0.05658193497608106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.0711111466089884, + "completions/mean_terminated_length": 1.0711111466089884, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04531929544173181, + "epoch": 0.29164926138305547, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.290667533874512, + "kd/policy_loss": -3.6875577608744305, + "kd/rkl_advantage_mean": 64.49456113179525, + "kd/rkl_advantage_p95": 84.94242649873098, + "kd/rkl_advantage_std": 34.248849747578305, + "kd/ssd_loss": 0.0, + "learning_rate": 7.08388713781187e-07, + "loss": -14.750229899088541, + "num_tokens": 105603793.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0110519051551818, + "sampling/importance_sampling_ratio/mean": 0.9994748016198476, + "sampling/importance_sampling_ratio/min": 0.8944540162881215, + "sampling/sampling_logp_difference/max": 0.1101405183939884, + "sampling/sampling_logp_difference/mean": 0.0018480188417015597, + "step": 7680, + "step_time": 4.461400881153532, + "student/entropy": 0.04531929544173181 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.233333333333333, + "completions/max_terminated_length": 6.233333333333333, + "completions/mean_length": 1.0516666928927103, + "completions/mean_terminated_length": 1.0516666928927103, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05065520256757736, + "epoch": 0.29278851631033304, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.46839714050293, + "kd/policy_loss": -1.5164430121580759, + "kd/rkl_advantage_mean": 51.374892775217695, + "kd/rkl_advantage_p95": 72.87530744870504, + "kd/rkl_advantage_std": 32.91867234955232, + "kd/ssd_loss": 0.0, + "learning_rate": 7.072494588539095e-07, + "loss": -6.065772501627604, + "num_tokens": 105901547.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.056874430179596, + "sampling/importance_sampling_ratio/mean": 1.000281322002411, + "sampling/importance_sampling_ratio/min": 0.9169671585162481, + "sampling/sampling_logp_difference/max": 0.10225106889071564, + "sampling/sampling_logp_difference/mean": 0.0018402611904699976, + "step": 7710, + "step_time": 4.315054225347315, + "student/entropy": 0.05065520256757736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.633333333333334, + "completions/max_terminated_length": 7.633333333333334, + "completions/mean_length": 1.1077778140703838, + "completions/mean_terminated_length": 1.1077778140703838, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0565589955697457, + "epoch": 0.2939277712376106, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.413103103637695, + "kd/policy_loss": -5.498162798086802, + "kd/rkl_advantage_mean": 70.74499637285868, + "kd/rkl_advantage_p95": 90.41675057411194, + "kd/rkl_advantage_std": 32.64632789095243, + "kd/ssd_loss": 0.0, + "learning_rate": 7.061102039266319e-07, + "loss": -21.992655436197918, + "num_tokens": 106215471.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.106408413251241, + "sampling/importance_sampling_ratio/mean": 0.9999673942724864, + "sampling/importance_sampling_ratio/min": 0.8786026964584986, + "sampling/sampling_logp_difference/max": 0.16483106141289075, + "sampling/sampling_logp_difference/mean": 0.002883138196193613, + "step": 7740, + "step_time": 4.412777659848022, + "student/entropy": 0.0565589955697457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 22.933333333333334, + "completions/max_terminated_length": 6.233333333333333, + "completions/mean_length": 1.1927778204282125, + "completions/mean_terminated_length": 1.0508567055066427, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.035480755396808186, + "epoch": 0.29506702616488817, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.497703552246094, + "kd/policy_loss": -1.8183518429597219, + "kd/rkl_advantage_mean": 54.26096092859904, + "kd/rkl_advantage_p95": 73.29245903491974, + "kd/rkl_advantage_std": 31.050885094205537, + "kd/ssd_loss": 0.0, + "learning_rate": 7.049709489993543e-07, + "loss": -7.273408508300781, + "num_tokens": 106529597.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0852419058481853, + "sampling/importance_sampling_ratio/mean": 0.9999605854352315, + "sampling/importance_sampling_ratio/min": 0.9026652574539185, + "sampling/sampling_logp_difference/max": 0.13669378499810894, + "sampling/sampling_logp_difference/mean": 0.0016238964187020126, + "step": 7770, + "step_time": 4.4548940496635625, + "student/entropy": 0.035480755396808186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.7, + "completions/max_terminated_length": 5.7, + "completions/mean_length": 1.060277811686198, + "completions/mean_terminated_length": 1.060277811686198, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.061690683787067734, + "epoch": 0.29620628109216574, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.651674270629883, + "kd/policy_loss": -2.1158337235450744, + "kd/rkl_advantage_mean": 49.455622339248656, + "kd/rkl_advantage_p95": 68.0070591211319, + "kd/rkl_advantage_std": 29.6381866812706, + "kd/ssd_loss": 0.0, + "learning_rate": 7.038316940720769e-07, + "loss": -8.463334147135416, + "num_tokens": 106838870.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0793308893839517, + "sampling/importance_sampling_ratio/mean": 1.0002151310443879, + "sampling/importance_sampling_ratio/min": 0.9250779529412587, + "sampling/sampling_logp_difference/max": 0.11912610835085312, + "sampling/sampling_logp_difference/mean": 0.00185256765204637, + "step": 7800, + "step_time": 4.322435985940198, + "student/entropy": 0.061690683787067734 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.766666666666667, + "completions/max_terminated_length": 7.766666666666667, + "completions/mean_length": 1.0913889209429424, + "completions/mean_terminated_length": 1.0913889209429424, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05807010978460312, + "epoch": 0.2973455360194433, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.524091720581055, + "kd/policy_loss": -4.889228383700053, + "kd/rkl_advantage_mean": 70.52988874117533, + "kd/rkl_advantage_p95": 90.4548765818278, + "kd/rkl_advantage_std": 33.85387809276581, + "kd/ssd_loss": 0.0, + "learning_rate": 7.026924391447993e-07, + "loss": -19.55691935221354, + "num_tokens": 107141935.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.020623298486074, + "sampling/importance_sampling_ratio/mean": 0.9996811350186666, + "sampling/importance_sampling_ratio/min": 0.8975313693284989, + "sampling/sampling_logp_difference/max": 0.14727196694972614, + "sampling/sampling_logp_difference/mean": 0.0026279794013438127, + "step": 7830, + "step_time": 4.428012685318633, + "student/entropy": 0.05807010978460312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.466666666666667, + "completions/max_terminated_length": 6.466666666666667, + "completions/mean_length": 1.0627778132756551, + "completions/mean_terminated_length": 1.0627778132756551, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0478731969371438, + "epoch": 0.29848479094672087, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 42.67565155029297, + "kd/policy_loss": -3.6076173543930055, + "kd/rkl_advantage_mean": 63.07207929293315, + "kd/rkl_advantage_p95": 84.43856172561645, + "kd/rkl_advantage_std": 34.7333481023709, + "kd/ssd_loss": 0.0, + "learning_rate": 7.015531842175218e-07, + "loss": -14.43046875, + "num_tokens": 107446929.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0114310383796692, + "sampling/importance_sampling_ratio/mean": 0.9998181939125061, + "sampling/importance_sampling_ratio/min": 0.9190414369106292, + "sampling/sampling_logp_difference/max": 0.0893996716166536, + "sampling/sampling_logp_difference/mean": 0.001414983868986989, + "step": 7860, + "step_time": 4.354090742328359, + "student/entropy": 0.0478731969371438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.233333333333333, + "completions/max_terminated_length": 6.233333333333333, + "completions/mean_length": 1.0650000373522441, + "completions/mean_terminated_length": 1.0650000373522441, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03252848273453613, + "epoch": 0.2996240458739984, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.492870330810547, + "kd/policy_loss": -2.8260395725568137, + "kd/rkl_advantage_mean": 56.86504179636638, + "kd/rkl_advantage_p95": 78.69718480904898, + "kd/rkl_advantage_std": 33.39612610340119, + "kd/ssd_loss": 0.0, + "learning_rate": 7.004139292902441e-07, + "loss": -11.304156494140624, + "num_tokens": 107746851.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0226401209831237, + "sampling/importance_sampling_ratio/mean": 0.9997349580128988, + "sampling/importance_sampling_ratio/min": 0.9273622353871663, + "sampling/sampling_logp_difference/max": 0.10060785016976297, + "sampling/sampling_logp_difference/mean": 0.0013032111373225538, + "step": 7890, + "step_time": 4.393577629475233, + "student/entropy": 0.03252848273453613 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.166666666666667, + "completions/max_terminated_length": 5.166666666666667, + "completions/mean_length": 1.0372222463289897, + "completions/mean_terminated_length": 1.0372222463289897, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03918233873943488, + "epoch": 0.30076330080127595, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.021451950073242, + "kd/policy_loss": -0.2950378576914469, + "kd/rkl_advantage_mean": 40.140770037968956, + "kd/rkl_advantage_p95": 61.729708250363664, + "kd/rkl_advantage_std": 30.226204154888787, + "kd/ssd_loss": 0.0, + "learning_rate": 6.992746743629666e-07, + "loss": -1.180152130126953, + "num_tokens": 108043649.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0109352548917134, + "sampling/importance_sampling_ratio/mean": 1.0001261413097382, + "sampling/importance_sampling_ratio/min": 0.9551431059837341, + "sampling/sampling_logp_difference/max": 0.05917818481102586, + "sampling/sampling_logp_difference/mean": 0.001089003363934656, + "step": 7920, + "step_time": 4.368706463153163, + "student/entropy": 0.03918233873943488 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.2, + "completions/max_terminated_length": 7.2, + "completions/mean_length": 1.070555587609609, + "completions/mean_terminated_length": 1.070555587609609, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05526240880911549, + "epoch": 0.3019025557285535, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.461528778076172, + "kd/policy_loss": -3.643688742319743, + "kd/rkl_advantage_mean": 64.60910952885946, + "kd/rkl_advantage_p95": 84.24497224489848, + "kd/rkl_advantage_std": 33.6750356088082, + "kd/ssd_loss": 0.0, + "learning_rate": 6.98135419435689e-07, + "loss": -14.574757893880209, + "num_tokens": 108358111.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0259116291999817, + "sampling/importance_sampling_ratio/mean": 1.0002074420452118, + "sampling/importance_sampling_ratio/min": 0.9318866769472758, + "sampling/sampling_logp_difference/max": 0.09578144786258538, + "sampling/sampling_logp_difference/mean": 0.001956306231052925, + "step": 7950, + "step_time": 4.4120097844395785, + "student/entropy": 0.05526240880911549 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.066666666666666, + "completions/max_terminated_length": 7.066666666666666, + "completions/mean_length": 1.0741666992505392, + "completions/mean_terminated_length": 1.0741666992505392, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05022117091963688, + "epoch": 0.3030418106558311, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 654.2015380859375, + "kd/policy_loss": -3.0375020225842793, + "kd/rkl_advantage_mean": 63.290783898035684, + "kd/rkl_advantage_p95": 84.57541292508444, + "kd/rkl_advantage_std": 35.63148854275544, + "kd/ssd_loss": 0.0, + "learning_rate": 6.969961645084114e-07, + "loss": -12.15000712076823, + "num_tokens": 108663266.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0122997601826986, + "sampling/importance_sampling_ratio/mean": 0.9994170864423116, + "sampling/importance_sampling_ratio/min": 0.8859634121259053, + "sampling/sampling_logp_difference/max": 0.10193618373014032, + "sampling/sampling_logp_difference/mean": 0.0018289954256033524, + "step": 7980, + "step_time": 4.269930806864674, + "student/entropy": 0.05022117091963688 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.2, + "completions/max_terminated_length": 5.2, + "completions/mean_length": 1.0447222550710042, + "completions/mean_terminated_length": 1.0447222550710042, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.042081597913056615, + "epoch": 0.30418106558310865, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.68841552734375, + "kd/policy_loss": -0.5689330498377482, + "kd/rkl_advantage_mean": 42.85566790898641, + "kd/rkl_advantage_p95": 67.13522117932638, + "kd/rkl_advantage_std": 32.65272638897101, + "kd/ssd_loss": 0.0, + "learning_rate": 6.958569095811339e-07, + "loss": -2.275731404622396, + "num_tokens": 108972747.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0559928973515829, + "sampling/importance_sampling_ratio/mean": 1.0000874201456706, + "sampling/importance_sampling_ratio/min": 0.926202396551768, + "sampling/sampling_logp_difference/max": 0.12087360795897742, + "sampling/sampling_logp_difference/mean": 0.0016447428439278155, + "step": 8010, + "step_time": 4.296660458129676, + "student/entropy": 0.042081597913056615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.333333333333333, + "completions/max_terminated_length": 6.333333333333333, + "completions/mean_length": 1.0630555828412374, + "completions/mean_terminated_length": 1.0630555828412374, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04479624123002092, + "epoch": 0.3053203205103862, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.701250076293945, + "kd/policy_loss": -2.8597429831822714, + "kd/rkl_advantage_mean": 55.49048248926798, + "kd/rkl_advantage_p95": 73.58624823093415, + "kd/rkl_advantage_std": 30.220402835806212, + "kd/ssd_loss": 0.0, + "learning_rate": 6.947176546538564e-07, + "loss": -11.438972981770833, + "num_tokens": 109294198.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0019338329633076, + "sampling/importance_sampling_ratio/mean": 0.9997166077295939, + "sampling/importance_sampling_ratio/min": 0.9427263975143433, + "sampling/sampling_logp_difference/max": 0.05677448019074897, + "sampling/sampling_logp_difference/mean": 0.0011194965064836046, + "step": 8040, + "step_time": 4.450798992614727, + "student/entropy": 0.04479624123002092 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0850000341733297, + "completions/mean_terminated_length": 1.0850000341733297, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06064065772419174, + "epoch": 0.3064595754376638, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 86.59954833984375, + "kd/policy_loss": -4.091134812434515, + "kd/rkl_advantage_mean": 63.82775123914083, + "kd/rkl_advantage_p95": 84.62879736423493, + "kd/rkl_advantage_std": 32.959139246741934, + "kd/ssd_loss": 0.0, + "learning_rate": 6.935783997265788e-07, + "loss": -16.36453857421875, + "num_tokens": 109595080.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0291709820429484, + "sampling/importance_sampling_ratio/mean": 0.9996165792147319, + "sampling/importance_sampling_ratio/min": 0.88343070546786, + "sampling/sampling_logp_difference/max": 0.14945894681538144, + "sampling/sampling_logp_difference/mean": 0.0025765008079664162, + "step": 8070, + "step_time": 4.254806600091979, + "student/entropy": 0.06064065772419174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.466666666666667, + "completions/max_terminated_length": 6.466666666666667, + "completions/mean_length": 1.0700000405311585, + "completions/mean_terminated_length": 1.0700000405311585, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04210305515055855, + "epoch": 0.30759883036494134, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.823507308959961, + "kd/policy_loss": -3.5766851166884104, + "kd/rkl_advantage_mean": 59.13591108322144, + "kd/rkl_advantage_p95": 79.22237093448639, + "kd/rkl_advantage_std": 31.631087653835614, + "kd/ssd_loss": 0.0, + "learning_rate": 6.924391447993012e-07, + "loss": -14.30673828125, + "num_tokens": 109902476.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0243328173955282, + "sampling/importance_sampling_ratio/mean": 0.9999161620934804, + "sampling/importance_sampling_ratio/min": 0.9394362409909566, + "sampling/sampling_logp_difference/max": 0.0810673555204024, + "sampling/sampling_logp_difference/mean": 0.0012743356521241367, + "step": 8100, + "step_time": 4.368599511950743, + "student/entropy": 0.04210305515055855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.433333333333334, + "completions/max_terminated_length": 5.433333333333334, + "completions/mean_length": 1.0494444648424783, + "completions/mean_terminated_length": 1.0494444648424783, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05477916793897748, + "epoch": 0.3087380852922189, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 85.27178955078125, + "kd/policy_loss": -1.0963461915651957, + "kd/rkl_advantage_mean": 40.81022582054138, + "kd/rkl_advantage_p95": 56.61218899885814, + "kd/rkl_advantage_std": 25.301554778218268, + "kd/ssd_loss": 0.0, + "learning_rate": 6.912998898720237e-07, + "loss": -4.385385131835937, + "num_tokens": 110206878.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0044513543446858, + "sampling/importance_sampling_ratio/mean": 0.999976662794749, + "sampling/importance_sampling_ratio/min": 0.9423550983270009, + "sampling/sampling_logp_difference/max": 0.08613739075760046, + "sampling/sampling_logp_difference/mean": 0.0014780881979580347, + "step": 8130, + "step_time": 4.300074052492467, + "student/entropy": 0.05477916793897748 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 23.466666666666665, + "completions/max_terminated_length": 6.433333333333334, + "completions/mean_length": 1.2005555907885233, + "completions/mean_terminated_length": 1.0586111346880596, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05250024565805991, + "epoch": 0.3098773402194964, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.514324188232422, + "kd/policy_loss": -2.1023642967144647, + "kd/rkl_advantage_mean": 59.3773858388265, + "kd/rkl_advantage_p95": 78.62536169687907, + "kd/rkl_advantage_std": 32.4946685269475, + "kd/ssd_loss": 0.0, + "learning_rate": 6.90160634944746e-07, + "loss": -8.409455362955729, + "num_tokens": 110509312.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0411577582359315, + "sampling/importance_sampling_ratio/mean": 0.9997992098331452, + "sampling/importance_sampling_ratio/min": 0.902098597586155, + "sampling/sampling_logp_difference/max": 0.1481880596994112, + "sampling/sampling_logp_difference/mean": 0.0017062394249175365, + "step": 8160, + "step_time": 4.3967555475072, + "student/entropy": 0.05250024565805991 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.633333333333334, + "completions/max_terminated_length": 7.633333333333334, + "completions/mean_length": 1.0825000445048014, + "completions/mean_terminated_length": 1.0825000445048014, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05424700301761429, + "epoch": 0.311016595146774, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.225374221801758, + "kd/policy_loss": -5.054486640294393, + "kd/rkl_advantage_mean": 74.24087931315104, + "kd/rkl_advantage_p95": 95.51826809247335, + "kd/rkl_advantage_std": 36.494769940773644, + "kd/ssd_loss": 0.0, + "learning_rate": 6.890213800174686e-07, + "loss": -20.217948404947915, + "num_tokens": 110824601.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.035466480255127, + "sampling/importance_sampling_ratio/mean": 0.9997628410657247, + "sampling/importance_sampling_ratio/min": 0.9146363317966462, + "sampling/sampling_logp_difference/max": 0.1461320730857551, + "sampling/sampling_logp_difference/mean": 0.0022533439235606543, + "step": 8190, + "step_time": 4.458789813615537, + "student/entropy": 0.05424700301761429 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.933333333333334, + "completions/max_terminated_length": 4.933333333333334, + "completions/mean_length": 1.0491666833559672, + "completions/mean_terminated_length": 1.0491666833559672, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04215664877556265, + "epoch": 0.31215585007405156, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.43132209777832, + "kd/policy_loss": 0.021862618128458657, + "kd/rkl_advantage_mean": 30.959268267949422, + "kd/rkl_advantage_p95": 44.93621680736542, + "kd/rkl_advantage_std": 21.707754121224085, + "kd/ssd_loss": 0.0, + "learning_rate": 6.87882125090191e-07, + "loss": 0.08745032946268717, + "num_tokens": 111145202.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0102949619293213, + "sampling/importance_sampling_ratio/mean": 0.9999871035416921, + "sampling/importance_sampling_ratio/min": 0.9609114309151967, + "sampling/sampling_logp_difference/max": 0.04805028622504324, + "sampling/sampling_logp_difference/mean": 0.0011390703366487287, + "step": 8220, + "step_time": 4.542926092422567, + "student/entropy": 0.04215664877556265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.866666666666667, + "completions/max_terminated_length": 8.866666666666667, + "completions/mean_length": 1.1197222590446472, + "completions/mean_terminated_length": 1.1197222590446472, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06132110661516587, + "epoch": 0.3132951050013291, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.971272468566895, + "kd/policy_loss": -6.741475240389506, + "kd/rkl_advantage_mean": 85.67139031092326, + "kd/rkl_advantage_p95": 107.53090866406758, + "kd/rkl_advantage_std": 38.61041963497798, + "kd/ssd_loss": 0.0, + "learning_rate": 6.867428701629134e-07, + "loss": -26.965899658203124, + "num_tokens": 111446985.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0827115019162497, + "sampling/importance_sampling_ratio/mean": 1.000008793671926, + "sampling/importance_sampling_ratio/min": 0.8844424605369567, + "sampling/sampling_logp_difference/max": 0.161715024880444, + "sampling/sampling_logp_difference/mean": 0.002853313227145312, + "step": 8250, + "step_time": 4.357503163670966, + "student/entropy": 0.06132110661516587 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.833333333333333, + "completions/max_terminated_length": 5.833333333333333, + "completions/mean_length": 1.0550000230471293, + "completions/mean_terminated_length": 1.0550000230471293, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.039581295537451905, + "epoch": 0.3144343599286067, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.69308853149414, + "kd/policy_loss": -1.0770691593488058, + "kd/rkl_advantage_mean": 43.513201999664304, + "kd/rkl_advantage_p95": 62.26142741044362, + "kd/rkl_advantage_std": 28.265736523270608, + "kd/ssd_loss": 0.0, + "learning_rate": 6.856036152356358e-07, + "loss": -4.308275858561198, + "num_tokens": 111750431.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.043679130077362, + "sampling/importance_sampling_ratio/mean": 1.0001331309477488, + "sampling/importance_sampling_ratio/min": 0.9329546853899956, + "sampling/sampling_logp_difference/max": 0.11570516621383528, + "sampling/sampling_logp_difference/mean": 0.0015501167537877337, + "step": 8280, + "step_time": 4.393070696736686, + "student/entropy": 0.039581295537451905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.633333333333334, + "completions/max_terminated_length": 6.633333333333334, + "completions/mean_length": 1.059166701634725, + "completions/mean_terminated_length": 1.059166701634725, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048789991283168396, + "epoch": 0.31557361485588425, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.982619285583496, + "kd/policy_loss": -3.0542729695638022, + "kd/rkl_advantage_mean": 62.877692222595215, + "kd/rkl_advantage_p95": 84.80019497871399, + "kd/rkl_advantage_std": 36.13868500987689, + "kd/ssd_loss": 0.0, + "learning_rate": 6.844643603083583e-07, + "loss": -12.217091878255209, + "num_tokens": 112056164.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.026712175210317, + "sampling/importance_sampling_ratio/mean": 1.000026124715805, + "sampling/importance_sampling_ratio/min": 0.9280485192934672, + "sampling/sampling_logp_difference/max": 0.08773839225371678, + "sampling/sampling_logp_difference/mean": 0.0015336366312112659, + "step": 8310, + "step_time": 4.474560501453622, + "student/entropy": 0.048789991283168396 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.533333333333333, + "completions/max_terminated_length": 5.533333333333333, + "completions/mean_length": 1.0427778045336404, + "completions/mean_terminated_length": 1.0427778045336404, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03873645399386684, + "epoch": 0.3167128697831618, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 101.49594116210938, + "kd/policy_loss": -0.6914379298686981, + "kd/rkl_advantage_mean": 45.358932701746625, + "kd/rkl_advantage_p95": 67.42357000509898, + "kd/rkl_advantage_std": 32.345272528628506, + "kd/ssd_loss": 0.0, + "learning_rate": 6.833251053810808e-07, + "loss": -2.7657511393229166, + "num_tokens": 112361966.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0238837401072185, + "sampling/importance_sampling_ratio/mean": 0.9999281426270803, + "sampling/importance_sampling_ratio/min": 0.9181015392144521, + "sampling/sampling_logp_difference/max": 0.0905059517826885, + "sampling/sampling_logp_difference/mean": 0.0013718782419649264, + "step": 8340, + "step_time": 4.391854410319744, + "student/entropy": 0.03873645399386684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.8, + "completions/max_terminated_length": 6.8, + "completions/mean_length": 1.0616667032241822, + "completions/mean_terminated_length": 1.0616667032241822, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06264925533905626, + "epoch": 0.3178521247104394, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.195463180541992, + "kd/policy_loss": -2.55622007449468, + "kd/rkl_advantage_mean": 62.935119247436525, + "kd/rkl_advantage_p95": 90.22391186555227, + "kd/rkl_advantage_std": 39.656147700548175, + "kd/ssd_loss": 0.0, + "learning_rate": 6.821858504538031e-07, + "loss": -10.224881998697917, + "num_tokens": 112665988.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0250489910443623, + "sampling/importance_sampling_ratio/mean": 0.9999535183111826, + "sampling/importance_sampling_ratio/min": 0.8992321292559305, + "sampling/sampling_logp_difference/max": 0.1425897104976078, + "sampling/sampling_logp_difference/mean": 0.002361572015797719, + "step": 8370, + "step_time": 4.393382909645637, + "student/entropy": 0.06264925533905626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.666666666666667, + "completions/max_terminated_length": 5.666666666666667, + "completions/mean_length": 1.0527777989705405, + "completions/mean_terminated_length": 1.0527777989705405, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04908504622677962, + "epoch": 0.31899137963771695, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 51.31776809692383, + "kd/policy_loss": -1.3176793177922568, + "kd/rkl_advantage_mean": 44.145345290501915, + "kd/rkl_advantage_p95": 61.807182606061296, + "kd/rkl_advantage_std": 27.76801250378291, + "kd/ssd_loss": 0.0, + "learning_rate": 6.810465955265257e-07, + "loss": -5.27071533203125, + "num_tokens": 112985434.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0727175951004029, + "sampling/importance_sampling_ratio/mean": 1.0006399432818094, + "sampling/importance_sampling_ratio/min": 0.9487148761749268, + "sampling/sampling_logp_difference/max": 0.10707876135905584, + "sampling/sampling_logp_difference/mean": 0.0017897734224485854, + "step": 8400, + "step_time": 4.572231657380083, + "student/entropy": 0.04908504622677962 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.133333333333334, + "completions/max_terminated_length": 7.133333333333334, + "completions/mean_length": 1.0622222463289896, + "completions/mean_terminated_length": 1.0622222463289896, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0659478577474753, + "epoch": 0.3201306345649945, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 31.045188903808594, + "kd/policy_loss": -2.675288426876068, + "kd/rkl_advantage_mean": 59.74862553278605, + "kd/rkl_advantage_p95": 79.05596103668213, + "kd/rkl_advantage_std": 32.8589763045311, + "kd/ssd_loss": 0.0, + "learning_rate": 6.799073405992481e-07, + "loss": -10.70115458170573, + "num_tokens": 113289794.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0170451124509177, + "sampling/importance_sampling_ratio/mean": 1.0002321461836496, + "sampling/importance_sampling_ratio/min": 0.9232628961404165, + "sampling/sampling_logp_difference/max": 0.09375334888075788, + "sampling/sampling_logp_difference/mean": 0.0019385486802396676, + "step": 8430, + "step_time": 4.364626038932086, + "student/entropy": 0.0659478577474753 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.733333333333333, + "completions/max_terminated_length": 5.733333333333333, + "completions/mean_length": 1.0516666889190673, + "completions/mean_terminated_length": 1.0516666889190673, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05489793714756767, + "epoch": 0.32126988949227203, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.788602828979492, + "kd/policy_loss": -0.45128427545229594, + "kd/rkl_advantage_mean": 43.54597495396932, + "kd/rkl_advantage_p95": 62.1268505970637, + "kd/rkl_advantage_std": 29.66359998981158, + "kd/ssd_loss": 0.0, + "learning_rate": 6.787680856719705e-07, + "loss": -1.8051373799641928, + "num_tokens": 113589372.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0138230641682944, + "sampling/importance_sampling_ratio/mean": 1.0001287897427877, + "sampling/importance_sampling_ratio/min": 0.9399310151735941, + "sampling/sampling_logp_difference/max": 0.07446074481122196, + "sampling/sampling_logp_difference/mean": 0.001295437269921725, + "step": 8460, + "step_time": 4.381445712204246, + "student/entropy": 0.05489793714756767 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.6, + "completions/max_terminated_length": 7.6, + "completions/mean_length": 1.0761111497879028, + "completions/mean_terminated_length": 1.0761111497879028, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05172098471472661, + "epoch": 0.3224091444195496, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.861440658569336, + "kd/policy_loss": -4.050652996699015, + "kd/rkl_advantage_mean": 72.5620075861613, + "kd/rkl_advantage_p95": 95.64459711710612, + "kd/rkl_advantage_std": 38.99508637487888, + "kd/ssd_loss": 0.0, + "learning_rate": 6.776288307446929e-07, + "loss": -16.20261433919271, + "num_tokens": 113895294.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.061428952217102, + "sampling/importance_sampling_ratio/mean": 1.0002863844235739, + "sampling/importance_sampling_ratio/min": 0.9077094415823619, + "sampling/sampling_logp_difference/max": 0.14317340875665346, + "sampling/sampling_logp_difference/mean": 0.0024899604070621234, + "step": 8490, + "step_time": 4.393870976377124, + "student/entropy": 0.05172098471472661 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.866666666666666, + "completions/max_terminated_length": 5.866666666666666, + "completions/mean_length": 1.0605555812517802, + "completions/mean_terminated_length": 1.0605555812517802, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04961898233741522, + "epoch": 0.32354839934682716, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.032922744750977, + "kd/policy_loss": -2.013447666168213, + "kd/rkl_advantage_mean": 43.17284212112427, + "kd/rkl_advantage_p95": 56.77803764343262, + "kd/rkl_advantage_std": 23.22616145511468, + "kd/ssd_loss": 0.0, + "learning_rate": 6.764895758174154e-07, + "loss": -8.05379130045573, + "num_tokens": 114205808.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0255163788795472, + "sampling/importance_sampling_ratio/mean": 1.0002156953016916, + "sampling/importance_sampling_ratio/min": 0.9360756138960521, + "sampling/sampling_logp_difference/max": 0.09735236621151368, + "sampling/sampling_logp_difference/mean": 0.0016641421912936493, + "step": 8520, + "step_time": 4.522610259832193, + "student/entropy": 0.04961898233741522 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.4, + "completions/max_terminated_length": 7.4, + "completions/mean_length": 1.0791667024294536, + "completions/mean_terminated_length": 1.0791667024294536, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.058180251313994326, + "epoch": 0.32468765427410473, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.533843994140625, + "kd/policy_loss": -4.0461149454116825, + "kd/rkl_advantage_mean": 69.05082518259684, + "kd/rkl_advantage_p95": 90.37921280860901, + "kd/rkl_advantage_std": 36.13824648658434, + "kd/ssd_loss": 0.0, + "learning_rate": 6.753503208901377e-07, + "loss": -16.184459431966147, + "num_tokens": 114509757.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0270339647928874, + "sampling/importance_sampling_ratio/mean": 1.0001550555229186, + "sampling/importance_sampling_ratio/min": 0.9044692277908325, + "sampling/sampling_logp_difference/max": 0.10101691428571939, + "sampling/sampling_logp_difference/mean": 0.0020260385955528665, + "step": 8550, + "step_time": 4.427465370126689, + "student/entropy": 0.058180251313994326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 21.933333333333334, + "completions/max_terminated_length": 4.9, + "completions/mean_length": 1.1894444783528646, + "completions/mean_terminated_length": 1.0475000222524007, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04249153837251166, + "epoch": 0.3258269092013823, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.280386924743652, + "kd/policy_loss": -0.8391991118590038, + "kd/rkl_advantage_mean": 40.82435186704, + "kd/rkl_advantage_p95": 56.11462237040202, + "kd/rkl_advantage_std": 24.167049434781074, + "kd/ssd_loss": 0.0, + "learning_rate": 6.742110659628603e-07, + "loss": -3.3567967732747395, + "num_tokens": 114820223.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0060109535853068, + "sampling/importance_sampling_ratio/mean": 0.999757198492686, + "sampling/importance_sampling_ratio/min": 0.9301863610744476, + "sampling/sampling_logp_difference/max": 0.07359927197297414, + "sampling/sampling_logp_difference/mean": 0.001778772446171691, + "step": 8580, + "step_time": 4.438141556763245, + "student/entropy": 0.04249153837251166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.633333333333334, + "completions/max_terminated_length": 7.633333333333334, + "completions/mean_length": 1.089722255865733, + "completions/mean_terminated_length": 1.089722255865733, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05915519033248226, + "epoch": 0.32696616412865986, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.835052490234375, + "kd/policy_loss": -4.0699426134427386, + "kd/rkl_advantage_mean": 69.40588372548422, + "kd/rkl_advantage_p95": 90.54872070153554, + "kd/rkl_advantage_std": 35.71778132716815, + "kd/ssd_loss": 0.0, + "learning_rate": 6.730718110355828e-07, + "loss": -16.279768880208334, + "num_tokens": 115131858.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0176051020622254, + "sampling/importance_sampling_ratio/mean": 0.9995345155398051, + "sampling/importance_sampling_ratio/min": 0.8753465175628662, + "sampling/sampling_logp_difference/max": 0.15302977239092191, + "sampling/sampling_logp_difference/mean": 0.0022357971659706286, + "step": 8610, + "step_time": 4.486090882805486, + "student/entropy": 0.05915519033248226 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.7, + "completions/max_terminated_length": 6.7, + "completions/mean_length": 1.0722222487131754, + "completions/mean_terminated_length": 1.0722222487131754, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05496352690582474, + "epoch": 0.32810541905593743, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.689966201782227, + "kd/policy_loss": -2.956813422838847, + "kd/rkl_advantage_mean": 59.486454757054645, + "kd/rkl_advantage_p95": 78.92032066186269, + "kd/rkl_advantage_std": 32.71891222099463, + "kd/ssd_loss": 0.0, + "learning_rate": 6.719325561083051e-07, + "loss": -11.827256266276041, + "num_tokens": 115429854.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0043143312136331, + "sampling/importance_sampling_ratio/mean": 0.9997776846090952, + "sampling/importance_sampling_ratio/min": 0.9109061857064565, + "sampling/sampling_logp_difference/max": 0.09391828829733034, + "sampling/sampling_logp_difference/mean": 0.001943936557897056, + "step": 8640, + "step_time": 4.337498327779273, + "student/entropy": 0.05496352690582474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.379310344827586, + "completions/max_terminated_length": 6.379310344827586, + "completions/mean_length": 1.054597727183638, + "completions/mean_terminated_length": 1.054597727183638, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.038673212687517035, + "epoch": 0.329244673983215, + "eval/gt_acc_batch": 0.000574712673651761, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.319948196411133, + "kd/policy_loss": -2.161893034803456, + "kd/rkl_advantage_mean": 56.28199112004247, + "kd/rkl_advantage_p95": 76.41736167874829, + "kd/rkl_advantage_std": 33.34134717439783, + "kd/ssd_loss": 0.0, + "learning_rate": 6.707933011810276e-07, + "loss": -8.35932108561198, + "num_tokens": 115727052.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.000574712673651761, + "rewards/gt_logging_reward/std": 0.004432960830885789, + "sampling/importance_sampling_ratio/max": 1.009714541764095, + "sampling/importance_sampling_ratio/mean": 0.999771510732585, + "sampling/importance_sampling_ratio/min": 0.9323461364055502, + "sampling/sampling_logp_difference/max": 0.09304304345478785, + "sampling/sampling_logp_difference/mean": 0.0013862880908257875, + "step": 8670, + "step_time": 4.342302860373942, + "student/entropy": 0.038673212687517035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.166666666666667, + "completions/max_terminated_length": 6.166666666666667, + "completions/mean_length": 1.061388913790385, + "completions/mean_terminated_length": 1.061388913790385, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04830756951123476, + "epoch": 0.33038392891049256, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 80.910888671875, + "kd/policy_loss": -2.7938244541486106, + "kd/rkl_advantage_mean": 51.60972941716512, + "kd/rkl_advantage_p95": 67.68955118656159, + "kd/rkl_advantage_std": 27.333711313207946, + "kd/ssd_loss": 0.0, + "learning_rate": 6.6965404625375e-07, + "loss": -11.17530008951823, + "num_tokens": 116029937.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0124223272005717, + "sampling/importance_sampling_ratio/mean": 0.9996499657630921, + "sampling/importance_sampling_ratio/min": 0.9081501692533493, + "sampling/sampling_logp_difference/max": 0.1041248841676861, + "sampling/sampling_logp_difference/mean": 0.0020902250330739965, + "step": 8700, + "step_time": 4.362313764422045, + "student/entropy": 0.04830756951123476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.333333333333333, + "completions/max_terminated_length": 6.333333333333333, + "completions/mean_length": 1.0772222558657327, + "completions/mean_terminated_length": 1.0772222558657327, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05041498836750786, + "epoch": 0.3315231838377701, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 58.54641342163086, + "kd/policy_loss": -3.4135521690050763, + "kd/rkl_advantage_mean": 55.91039652824402, + "kd/rkl_advantage_p95": 73.06502104600271, + "kd/rkl_advantage_std": 28.666433853904405, + "kd/ssd_loss": 0.0, + "learning_rate": 6.685147913264724e-07, + "loss": -13.654210408528646, + "num_tokens": 116337423.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.039984941482544, + "sampling/importance_sampling_ratio/mean": 0.9999654670556386, + "sampling/importance_sampling_ratio/min": 0.9162278513113657, + "sampling/sampling_logp_difference/max": 0.10780058290498952, + "sampling/sampling_logp_difference/mean": 0.0018199398415163159, + "step": 8730, + "step_time": 4.496546982535317, + "student/entropy": 0.05041498836750786 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.233333333333333, + "completions/max_terminated_length": 6.233333333333333, + "completions/mean_length": 1.0575000246365864, + "completions/mean_terminated_length": 1.0575000246365864, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04757598523671428, + "epoch": 0.33266243876504764, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.779409408569336, + "kd/policy_loss": -1.86204883257548, + "kd/rkl_advantage_mean": 50.28779724438985, + "kd/rkl_advantage_p95": 67.55905416806539, + "kd/rkl_advantage_std": 28.963018877307572, + "kd/ssd_loss": 0.0, + "learning_rate": 6.673755363991948e-07, + "loss": -7.448193359375, + "num_tokens": 116638630.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0105928937594095, + "sampling/importance_sampling_ratio/mean": 0.9997288386027018, + "sampling/importance_sampling_ratio/min": 0.9172814687093099, + "sampling/sampling_logp_difference/max": 0.07534998732929428, + "sampling/sampling_logp_difference/mean": 0.0015517039069284995, + "step": 8760, + "step_time": 4.416808362945448, + "student/entropy": 0.04757598523671428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0608333587646483, + "completions/mean_terminated_length": 1.0608333587646483, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052601719244072837, + "epoch": 0.3338016936923252, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 26.65150260925293, + "kd/policy_loss": -1.8991596341133117, + "kd/rkl_advantage_mean": 50.15096302032471, + "kd/rkl_advantage_p95": 67.92067910035452, + "kd/rkl_advantage_std": 29.15325395564238, + "kd/ssd_loss": 0.0, + "learning_rate": 6.662362814719174e-07, + "loss": -7.59664306640625, + "num_tokens": 116944009.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0272071997324626, + "sampling/importance_sampling_ratio/mean": 0.9999563038349152, + "sampling/importance_sampling_ratio/min": 0.9050012280543646, + "sampling/sampling_logp_difference/max": 0.101736269146204, + "sampling/sampling_logp_difference/mean": 0.001951916680748885, + "step": 8790, + "step_time": 4.410293689187771, + "student/entropy": 0.052601719244072837 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.5, + "completions/max_terminated_length": 6.5, + "completions/mean_length": 1.078611143430074, + "completions/mean_terminated_length": 1.078611143430074, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0630355492544671, + "epoch": 0.3349409486196028, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 77.26055908203125, + "kd/policy_loss": -3.1338364839553834, + "kd/rkl_advantage_mean": 52.99015522003174, + "kd/rkl_advantage_p95": 68.16898012955984, + "kd/rkl_advantage_std": 26.181133898099265, + "kd/ssd_loss": 0.0, + "learning_rate": 6.650970265446398e-07, + "loss": -12.535343424479167, + "num_tokens": 117249900.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0191490451494853, + "sampling/importance_sampling_ratio/mean": 0.9999690632025401, + "sampling/importance_sampling_ratio/min": 0.9204579591751099, + "sampling/sampling_logp_difference/max": 0.08073435615127285, + "sampling/sampling_logp_difference/mean": 0.002071290307988723, + "step": 8820, + "step_time": 4.376642389191935, + "student/entropy": 0.0630355492544671 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.133333333333334, + "completions/max_terminated_length": 7.133333333333334, + "completions/mean_length": 1.0775000333786011, + "completions/mean_terminated_length": 1.0775000333786011, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05793803942700227, + "epoch": 0.33608020354688034, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 62.567420959472656, + "kd/policy_loss": -3.9071887969970702, + "kd/rkl_advantage_mean": 61.16843996047974, + "kd/rkl_advantage_p95": 79.06692927678426, + "kd/rkl_advantage_std": 30.312174209952353, + "kd/ssd_loss": 0.0, + "learning_rate": 6.639577716173622e-07, + "loss": -15.628754679361979, + "num_tokens": 117564107.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0477080583572387, + "sampling/importance_sampling_ratio/mean": 0.9999799370765686, + "sampling/importance_sampling_ratio/min": 0.8924432595570883, + "sampling/sampling_logp_difference/max": 0.11783248670399189, + "sampling/sampling_logp_difference/mean": 0.00219204709380089, + "step": 8850, + "step_time": 4.458272983172598, + "student/entropy": 0.05793803942700227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.133333333333333, + "completions/max_terminated_length": 8.133333333333333, + "completions/mean_length": 1.0947222510973613, + "completions/mean_terminated_length": 1.0947222510973613, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06880789908270041, + "epoch": 0.3372194584741579, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 28.753068923950195, + "kd/policy_loss": -4.546155130863189, + "kd/rkl_advantage_mean": 70.06152680714925, + "kd/rkl_advantage_p95": 90.31593416531881, + "kd/rkl_advantage_std": 33.87890353798866, + "kd/ssd_loss": 0.0, + "learning_rate": 6.628185166900847e-07, + "loss": -18.184619140625, + "num_tokens": 117871328.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.050211989879608, + "sampling/importance_sampling_ratio/mean": 1.0002931873003642, + "sampling/importance_sampling_ratio/min": 0.9312147160371145, + "sampling/sampling_logp_difference/max": 0.12074751037483414, + "sampling/sampling_logp_difference/mean": 0.002393970815076803, + "step": 8880, + "step_time": 4.401396637095604, + "student/entropy": 0.06880789908270041 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 22.5, + "completions/max_terminated_length": 5.733333333333333, + "completions/mean_length": 1.1958333730697632, + "completions/mean_terminated_length": 1.0539075930913289, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.039960889999444284, + "epoch": 0.3383587134014355, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.611350059509277, + "kd/policy_loss": -2.1187010327974956, + "kd/rkl_advantage_mean": 50.80792711575826, + "kd/rkl_advantage_p95": 67.69326576391856, + "kd/rkl_advantage_std": 27.445993050932884, + "kd/ssd_loss": 0.0, + "learning_rate": 6.616792617628071e-07, + "loss": -8.474802652994791, + "num_tokens": 118179521.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0516683300336203, + "sampling/importance_sampling_ratio/mean": 1.0002089242140453, + "sampling/importance_sampling_ratio/min": 0.9341693600018819, + "sampling/sampling_logp_difference/max": 0.0654296116437763, + "sampling/sampling_logp_difference/mean": 0.0011110639694379642, + "step": 8910, + "step_time": 4.524625773196264, + "student/entropy": 0.039960889999444284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 23.666666666666668, + "completions/max_terminated_length": 6.9, + "completions/mean_length": 1.2130555907885234, + "completions/mean_terminated_length": 1.071129810810089, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0538500456760327, + "epoch": 0.33949796832871304, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 26.795787811279297, + "kd/policy_loss": -2.8866783042748767, + "kd/rkl_advantage_mean": 63.775546836853025, + "kd/rkl_advantage_p95": 84.38305586179098, + "kd/rkl_advantage_std": 33.976305323839185, + "kd/ssd_loss": 0.0, + "learning_rate": 6.605400068355295e-07, + "loss": -11.546712239583334, + "num_tokens": 118478376.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0177515983581542, + "sampling/importance_sampling_ratio/mean": 0.9997077147165935, + "sampling/importance_sampling_ratio/min": 0.8940236320098242, + "sampling/sampling_logp_difference/max": 0.08879310886065166, + "sampling/sampling_logp_difference/mean": 0.0016707335598766804, + "step": 8940, + "step_time": 4.414979871824229, + "student/entropy": 0.0538500456760327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.466666666666667, + "completions/max_terminated_length": 6.466666666666667, + "completions/mean_length": 1.0666666905085245, + "completions/mean_terminated_length": 1.0666666905085245, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04140299897020062, + "epoch": 0.3406372232559906, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.42758560180664, + "kd/policy_loss": -2.8548290014266966, + "kd/rkl_advantage_mean": 52.18245533307393, + "kd/rkl_advantage_p95": 67.72761143843333, + "kd/rkl_advantage_std": 26.967234447598457, + "kd/ssd_loss": 0.0, + "learning_rate": 6.59400751908252e-07, + "loss": -11.419315592447917, + "num_tokens": 118778928.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0044655243555705, + "sampling/importance_sampling_ratio/mean": 0.9995116829872132, + "sampling/importance_sampling_ratio/min": 0.9034073412418365, + "sampling/sampling_logp_difference/max": 0.08920943763417502, + "sampling/sampling_logp_difference/mean": 0.0015298455216300984, + "step": 8970, + "step_time": 4.410022998725375, + "student/entropy": 0.04140299897020062 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.062500031789144, + "completions/mean_terminated_length": 1.062500031789144, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05319086660941442, + "epoch": 0.3417764781832681, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.886642456054688, + "kd/policy_loss": -1.7064401189486185, + "kd/rkl_advantage_mean": 51.53455109596253, + "kd/rkl_advantage_p95": 72.8699414173762, + "kd/rkl_advantage_std": 32.79965257644653, + "kd/ssd_loss": 0.0, + "learning_rate": 6.582614969809745e-07, + "loss": -6.8257603963216145, + "num_tokens": 119084113.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0212382038434347, + "sampling/importance_sampling_ratio/mean": 0.9999005993207296, + "sampling/importance_sampling_ratio/min": 0.9183120330174764, + "sampling/sampling_logp_difference/max": 0.09059387263841927, + "sampling/sampling_logp_difference/mean": 0.001510447386923867, + "step": 9000, + "step_time": 4.41080475319565, + "student/entropy": 0.05319086660941442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.066666666666666, + "completions/max_terminated_length": 7.066666666666666, + "completions/mean_length": 1.0800000270207724, + "completions/mean_terminated_length": 1.0800000270207724, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.056839726368586224, + "epoch": 0.3429157331105457, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 29.44142723083496, + "kd/policy_loss": -3.5246834993362426, + "kd/rkl_advantage_mean": 57.30963897705078, + "kd/rkl_advantage_p95": 73.31675469080606, + "kd/rkl_advantage_std": 27.991818192601205, + "kd/ssd_loss": 0.0, + "learning_rate": 6.571222420536968e-07, + "loss": -14.098734537760416, + "num_tokens": 119400129.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.068299949169159, + "sampling/importance_sampling_ratio/mean": 1.0003802676995595, + "sampling/importance_sampling_ratio/min": 0.9269150793552399, + "sampling/sampling_logp_difference/max": 0.11837554865827163, + "sampling/sampling_logp_difference/mean": 0.0021030595351476224, + "step": 9030, + "step_time": 4.5236043199702785, + "student/entropy": 0.056839726368586224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.266666666666667, + "completions/max_terminated_length": 8.266666666666667, + "completions/mean_length": 1.0875000397364298, + "completions/mean_terminated_length": 1.0875000397364298, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04646655083633959, + "epoch": 0.34405498803782325, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.229528427124023, + "kd/policy_loss": -4.8124965786933895, + "kd/rkl_advantage_mean": 81.38230619430541, + "kd/rkl_advantage_p95": 107.33219768206278, + "kd/rkl_advantage_std": 43.36672430932522, + "kd/ssd_loss": 0.0, + "learning_rate": 6.559829871264193e-07, + "loss": -19.24998575846354, + "num_tokens": 119715428.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0691664695739747, + "sampling/importance_sampling_ratio/mean": 0.9999178489049275, + "sampling/importance_sampling_ratio/min": 0.8924905906120936, + "sampling/sampling_logp_difference/max": 0.14517979761585592, + "sampling/sampling_logp_difference/mean": 0.0021903515628461418, + "step": 9060, + "step_time": 4.567149826061601, + "student/entropy": 0.04646655083633959 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.833333333333333, + "completions/max_terminated_length": 4.833333333333333, + "completions/mean_length": 1.0458333571751912, + "completions/mean_terminated_length": 1.0458333571751912, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0398841998539865, + "epoch": 0.3451942429651008, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 30.900976181030273, + "kd/policy_loss": -0.6274787584940592, + "kd/rkl_advantage_mean": 34.605816729863484, + "kd/rkl_advantage_p95": 50.93203605810801, + "kd/rkl_advantage_std": 23.662690337498983, + "kd/ssd_loss": 0.0, + "learning_rate": 6.548437321991417e-07, + "loss": -2.5099161783854167, + "num_tokens": 120028801.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.03896244764328, + "sampling/importance_sampling_ratio/mean": 1.0002184510231018, + "sampling/importance_sampling_ratio/min": 0.9482159912586212, + "sampling/sampling_logp_difference/max": 0.0958145797951147, + "sampling/sampling_logp_difference/mean": 0.0015924366288042316, + "step": 9090, + "step_time": 4.565424366062507, + "student/entropy": 0.0398841998539865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.833333333333334, + "completions/max_terminated_length": 8.833333333333334, + "completions/mean_length": 1.0886111458142598, + "completions/mean_terminated_length": 1.0886111458142598, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0602484958867232, + "epoch": 0.3463334978923784, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.648195266723633, + "kd/policy_loss": -5.195748178164164, + "kd/rkl_advantage_mean": 82.53276357650756, + "kd/rkl_advantage_p95": 107.50152281125386, + "kd/rkl_advantage_std": 41.58792796730995, + "kd/ssd_loss": 0.0, + "learning_rate": 6.537044772718641e-07, + "loss": -20.78299560546875, + "num_tokens": 120338512.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0413389921188354, + "sampling/importance_sampling_ratio/mean": 0.9997632841269175, + "sampling/importance_sampling_ratio/min": 0.8775921414295832, + "sampling/sampling_logp_difference/max": 0.1517658997947971, + "sampling/sampling_logp_difference/mean": 0.002714627170159171, + "step": 9120, + "step_time": 4.47983463278894, + "student/entropy": 0.0602484958867232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0572222471237183, + "completions/mean_terminated_length": 1.0572222471237183, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.039177017938345673, + "epoch": 0.34747275281965595, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.713775634765625, + "kd/policy_loss": -1.75442084868749, + "kd/rkl_advantage_mean": 54.658040618896486, + "kd/rkl_advantage_p95": 73.56063323020935, + "kd/rkl_advantage_std": 32.26571892996629, + "kd/ssd_loss": 0.0, + "learning_rate": 6.525652223445866e-07, + "loss": -7.017682902018229, + "num_tokens": 120641238.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0582042535146077, + "sampling/importance_sampling_ratio/mean": 1.0002158105373382, + "sampling/importance_sampling_ratio/min": 0.9247504293918609, + "sampling/sampling_logp_difference/max": 0.10762854610414554, + "sampling/sampling_logp_difference/mean": 0.0015670970732268566, + "step": 9150, + "step_time": 4.356374320395601, + "student/entropy": 0.039177017938345673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.833333333333333, + "completions/max_terminated_length": 6.833333333333333, + "completions/mean_length": 1.0666666944821677, + "completions/mean_terminated_length": 1.0666666944821677, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04561509300644199, + "epoch": 0.3486120077469335, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.626842498779297, + "kd/policy_loss": -2.5977643986543018, + "kd/rkl_advantage_mean": 58.35720101992289, + "kd/rkl_advantage_p95": 79.25827107429504, + "kd/rkl_advantage_std": 33.387100739280385, + "kd/ssd_loss": 0.0, + "learning_rate": 6.514259674173091e-07, + "loss": -10.391058349609375, + "num_tokens": 120941998.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0362886508305869, + "sampling/importance_sampling_ratio/mean": 0.9998700400193532, + "sampling/importance_sampling_ratio/min": 0.9165029287338257, + "sampling/sampling_logp_difference/max": 0.11508412055360774, + "sampling/sampling_logp_difference/mean": 0.0019614047914122542, + "step": 9180, + "step_time": 4.381530858894499, + "student/entropy": 0.04561509300644199 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.8, + "completions/max_terminated_length": 6.8, + "completions/mean_length": 1.054722253481547, + "completions/mean_terminated_length": 1.054722253481547, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048532754989961784, + "epoch": 0.3497512626742111, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 28.493167877197266, + "kd/policy_loss": -1.5454060633977253, + "kd/rkl_advantage_mean": 58.48483711878459, + "kd/rkl_advantage_p95": 84.5991318543752, + "kd/rkl_advantage_std": 38.85063316424688, + "kd/ssd_loss": 0.0, + "learning_rate": 6.502867124900316e-07, + "loss": -6.181624348958334, + "num_tokens": 121245723.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.1197587450345357, + "sampling/importance_sampling_ratio/mean": 1.000826930999756, + "sampling/importance_sampling_ratio/min": 0.9502125898996989, + "sampling/sampling_logp_difference/max": 0.14289280553348363, + "sampling/sampling_logp_difference/mean": 0.0020157531136646865, + "step": 9210, + "step_time": 4.459252396776962, + "student/entropy": 0.048532754989961784 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.2, + "completions/max_terminated_length": 7.2, + "completions/mean_length": 1.0641666889190673, + "completions/mean_terminated_length": 1.0641666889190673, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048014749151964985, + "epoch": 0.35089051760148865, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 37.45443344116211, + "kd/policy_loss": -2.5270143946011863, + "kd/rkl_advantage_mean": 59.28650353749593, + "kd/rkl_advantage_p95": 78.87249979178111, + "kd/rkl_advantage_std": 33.06051863034566, + "kd/ssd_loss": 0.0, + "learning_rate": 6.491474575627539e-07, + "loss": -10.108056640625, + "num_tokens": 121557082.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0544601400693259, + "sampling/importance_sampling_ratio/mean": 1.000432735681534, + "sampling/importance_sampling_ratio/min": 0.9501668035984039, + "sampling/sampling_logp_difference/max": 0.08860869786391655, + "sampling/sampling_logp_difference/mean": 0.0016324248659657314, + "step": 9240, + "step_time": 4.452491411981949, + "student/entropy": 0.048014749151964985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 23.6, + "completions/max_terminated_length": 6.933333333333334, + "completions/mean_length": 1.2027778069178263, + "completions/mean_terminated_length": 1.0608637054761252, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052800781869639955, + "epoch": 0.35202977252876616, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 56.14330291748047, + "kd/policy_loss": -1.9389148275057475, + "kd/rkl_advantage_mean": 59.30236512819926, + "kd/rkl_advantage_p95": 79.13082630634308, + "kd/rkl_advantage_std": 33.256845338145894, + "kd/ssd_loss": 0.0, + "learning_rate": 6.480082026354764e-07, + "loss": -7.7556610107421875, + "num_tokens": 121860868.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0853250980377198, + "sampling/importance_sampling_ratio/mean": 1.0005804657936097, + "sampling/importance_sampling_ratio/min": 0.9182059268156687, + "sampling/sampling_logp_difference/max": 0.12083203780154388, + "sampling/sampling_logp_difference/mean": 0.0018288169550942257, + "step": 9270, + "step_time": 4.4606919024644105, + "student/entropy": 0.052800781869639955 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.733333333333333, + "completions/max_terminated_length": 6.733333333333333, + "completions/mean_length": 1.0738889177640278, + "completions/mean_terminated_length": 1.0738889177640278, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0626379696962734, + "epoch": 0.3531690274560437, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 31.06585121154785, + "kd/policy_loss": -2.5794416069984436, + "kd/rkl_advantage_mean": 53.38811211585998, + "kd/rkl_advantage_p95": 73.10096576213837, + "kd/rkl_advantage_std": 30.318740617235502, + "kd/ssd_loss": 0.0, + "learning_rate": 6.468689477081988e-07, + "loss": -10.317767333984374, + "num_tokens": 122170014.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0651676376660666, + "sampling/importance_sampling_ratio/mean": 1.0005315999190012, + "sampling/importance_sampling_ratio/min": 0.9268592337767283, + "sampling/sampling_logp_difference/max": 0.11968602232324581, + "sampling/sampling_logp_difference/mean": 0.0025830596588396777, + "step": 9300, + "step_time": 4.364726761010631, + "student/entropy": 0.0626379696962734 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.166666666666666, + "completions/max_terminated_length": 8.166666666666666, + "completions/mean_length": 1.1008333762486775, + "completions/mean_terminated_length": 1.1008333762486775, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05655894322941701, + "epoch": 0.3543082823833213, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.135454177856445, + "kd/policy_loss": -4.6323785026868185, + "kd/rkl_advantage_mean": 76.95019402503968, + "kd/rkl_advantage_p95": 101.67440735499063, + "kd/rkl_advantage_std": 40.43474176426729, + "kd/ssd_loss": 0.0, + "learning_rate": 6.457296927809212e-07, + "loss": -18.529512532552083, + "num_tokens": 122479929.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0599230607350667, + "sampling/importance_sampling_ratio/mean": 0.9997828046480814, + "sampling/importance_sampling_ratio/min": 0.880991812547048, + "sampling/sampling_logp_difference/max": 0.16469952072948219, + "sampling/sampling_logp_difference/mean": 0.0027284908234529817, + "step": 9330, + "step_time": 4.519796724764941, + "student/entropy": 0.05655894322941701 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.966666666666667, + "completions/max_terminated_length": 5.966666666666667, + "completions/mean_length": 1.0577778021494548, + "completions/mean_terminated_length": 1.0577778021494548, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.040022356032083435, + "epoch": 0.35544753731059886, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 53.888301849365234, + "kd/policy_loss": -1.4872527639071147, + "kd/rkl_advantage_mean": 45.584127807617186, + "kd/rkl_advantage_p95": 62.07531192302704, + "kd/rkl_advantage_std": 27.096697602669398, + "kd/ssd_loss": 0.0, + "learning_rate": 6.445904378536436e-07, + "loss": -5.949009195963542, + "num_tokens": 122794801.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.028798266251882, + "sampling/importance_sampling_ratio/mean": 1.000060514609019, + "sampling/importance_sampling_ratio/min": 0.953876543045044, + "sampling/sampling_logp_difference/max": 0.07719238083809614, + "sampling/sampling_logp_difference/mean": 0.0013743478251853959, + "step": 9360, + "step_time": 4.450013736988573, + "student/entropy": 0.040022356032083435 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.7, + "completions/max_terminated_length": 6.7, + "completions/mean_length": 1.0669444719950358, + "completions/mean_terminated_length": 1.0669444719950358, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0519692805595696, + "epoch": 0.3565867922378764, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.63241958618164, + "kd/policy_loss": -2.5844037771224975, + "kd/rkl_advantage_mean": 55.616330003738405, + "kd/rkl_advantage_p95": 73.3194449742635, + "kd/rkl_advantage_std": 30.130801202853522, + "kd/ssd_loss": 0.0, + "learning_rate": 6.434511829263662e-07, + "loss": -10.337615966796875, + "num_tokens": 123103178.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.046026313304901, + "sampling/importance_sampling_ratio/mean": 1.0003112157185872, + "sampling/importance_sampling_ratio/min": 0.947564431031545, + "sampling/sampling_logp_difference/max": 0.08614550263931355, + "sampling/sampling_logp_difference/mean": 0.001716865602065809, + "step": 9390, + "step_time": 4.489692819584161, + "student/entropy": 0.0519692805595696 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.2, + "completions/max_terminated_length": 8.2, + "completions/mean_length": 1.0744444727897644, + "completions/mean_terminated_length": 1.0744444727897644, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.050386173324659465, + "epoch": 0.357726047165154, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.626453399658203, + "kd/policy_loss": -3.870876467227936, + "kd/rkl_advantage_mean": 73.7979968547821, + "kd/rkl_advantage_p95": 96.23450323740641, + "kd/rkl_advantage_std": 38.93591874241829, + "kd/ssd_loss": 0.0, + "learning_rate": 6.423119279990886e-07, + "loss": -15.483507283528645, + "num_tokens": 123413702.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0337229172388713, + "sampling/importance_sampling_ratio/mean": 0.9999932030836741, + "sampling/importance_sampling_ratio/min": 0.9233877976735433, + "sampling/sampling_logp_difference/max": 0.09851969826656083, + "sampling/sampling_logp_difference/mean": 0.002037753563490696, + "step": 9420, + "step_time": 4.448241786813984, + "student/entropy": 0.050386173324659465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.1, + "completions/max_terminated_length": 5.1, + "completions/mean_length": 1.0508333524068196, + "completions/mean_terminated_length": 1.0508333524068196, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05005316169311603, + "epoch": 0.35886530209243156, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.96539306640625, + "kd/policy_loss": -0.008547735214233399, + "kd/rkl_advantage_mean": 31.745315392812092, + "kd/rkl_advantage_p95": 44.92414108117421, + "kd/rkl_advantage_std": 21.593439862132072, + "kd/ssd_loss": 0.0, + "learning_rate": 6.41172673071811e-07, + "loss": -0.034189184506734215, + "num_tokens": 123726253.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.02422593832016, + "sampling/importance_sampling_ratio/mean": 0.9999278247356415, + "sampling/importance_sampling_ratio/min": 0.9358481347560883, + "sampling/sampling_logp_difference/max": 0.06902145226486027, + "sampling/sampling_logp_difference/mean": 0.0014568918733857573, + "step": 9450, + "step_time": 4.4811138098555, + "student/entropy": 0.05005316169311603 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.666666666666666, + "completions/max_terminated_length": 8.666666666666666, + "completions/mean_length": 1.1094444870948792, + "completions/mean_terminated_length": 1.1094444870948792, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.08470414805536469, + "epoch": 0.3600045570197091, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.689982414245605, + "kd/policy_loss": -6.812669386466344, + "kd/rkl_advantage_mean": 82.90691534678142, + "kd/rkl_advantage_p95": 107.42380531628926, + "kd/rkl_advantage_std": 38.17448086043199, + "kd/ssd_loss": 0.0, + "learning_rate": 6.400334181445335e-07, + "loss": -27.250679524739585, + "num_tokens": 124031543.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.081662154197693, + "sampling/importance_sampling_ratio/mean": 1.0001783510049185, + "sampling/importance_sampling_ratio/min": 0.8635458528995514, + "sampling/sampling_logp_difference/max": 0.18006258476525544, + "sampling/sampling_logp_difference/mean": 0.0037919626493627826, + "step": 9480, + "step_time": 4.408032074015742, + "student/entropy": 0.08470414805536469 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0438889106114706, + "completions/mean_terminated_length": 1.0438889106114706, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05196041852856676, + "epoch": 0.3611438119469867, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.439077377319336, + "kd/policy_loss": -0.48561687469482423, + "kd/rkl_advantage_mean": 46.510732364654544, + "kd/rkl_advantage_p95": 67.83980267047882, + "kd/rkl_advantage_std": 32.379494202136996, + "kd/ssd_loss": 0.0, + "learning_rate": 6.388941632172558e-07, + "loss": -1.9424667358398438, + "num_tokens": 124339381.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0113277594248453, + "sampling/importance_sampling_ratio/mean": 0.9998051424821218, + "sampling/importance_sampling_ratio/min": 0.9160185893376668, + "sampling/sampling_logp_difference/max": 0.1026800349354744, + "sampling/sampling_logp_difference/mean": 0.0015856888174312189, + "step": 9510, + "step_time": 4.491760816265984, + "student/entropy": 0.05196041852856676 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.0519444664319357, + "completions/mean_terminated_length": 1.0519444664319357, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.041177338423828286, + "epoch": 0.3622830668742642, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 33.30337905883789, + "kd/policy_loss": -0.9849346995353698, + "kd/rkl_advantage_mean": 52.80567161242167, + "kd/rkl_advantage_p95": 73.54073063532512, + "kd/rkl_advantage_std": 34.03736130942901, + "kd/ssd_loss": 0.0, + "learning_rate": 6.377549082899783e-07, + "loss": -3.9397427876790365, + "num_tokens": 124653576.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.010609213511149, + "sampling/importance_sampling_ratio/mean": 0.9997512896855673, + "sampling/importance_sampling_ratio/min": 0.9323034505049388, + "sampling/sampling_logp_difference/max": 0.09683727604957919, + "sampling/sampling_logp_difference/mean": 0.0014110319869359955, + "step": 9540, + "step_time": 4.52787892005872, + "student/entropy": 0.041177338423828286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.166666666666667, + "completions/max_terminated_length": 7.166666666666667, + "completions/mean_length": 1.091111143430074, + "completions/mean_terminated_length": 1.091111143430074, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0636802445165813, + "epoch": 0.36342232180154177, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.779733657836914, + "kd/policy_loss": -3.6873924255371096, + "kd/rkl_advantage_mean": 61.14431953430176, + "kd/rkl_advantage_p95": 78.91608417828878, + "kd/rkl_advantage_std": 30.65925998042027, + "kd/ssd_loss": 0.0, + "learning_rate": 6.366156533627008e-07, + "loss": -14.749568684895833, + "num_tokens": 124956816.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0242695252100626, + "sampling/importance_sampling_ratio/mean": 0.9997633894284567, + "sampling/importance_sampling_ratio/min": 0.9143672049045563, + "sampling/sampling_logp_difference/max": 0.09105518609285354, + "sampling/sampling_logp_difference/mean": 0.0020608669903595, + "step": 9570, + "step_time": 4.458033515202502, + "student/entropy": 0.0636802445165813 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.2, + "completions/max_terminated_length": 7.2, + "completions/mean_length": 1.076388919353485, + "completions/mean_terminated_length": 1.076388919353485, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04748968339214722, + "epoch": 0.36456157672881934, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 52.085506439208984, + "kd/policy_loss": -3.335078652699788, + "kd/rkl_advantage_mean": 61.10312218666077, + "kd/rkl_advantage_p95": 79.21516156991323, + "kd/rkl_advantage_std": 31.389937643210093, + "kd/ssd_loss": 0.0, + "learning_rate": 6.354763984354232e-07, + "loss": -13.340315755208334, + "num_tokens": 125264883.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0282063047091166, + "sampling/importance_sampling_ratio/mean": 0.9998441398143768, + "sampling/importance_sampling_ratio/min": 0.9223712960879008, + "sampling/sampling_logp_difference/max": 0.09471817426383496, + "sampling/sampling_logp_difference/mean": 0.0017324180817619586, + "step": 9600, + "step_time": 4.419823505504367, + "student/entropy": 0.04748968339214722 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.633333333333334, + "completions/max_terminated_length": 7.633333333333334, + "completions/mean_length": 1.066111139456431, + "completions/mean_terminated_length": 1.066111139456431, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04545215287556251, + "epoch": 0.3657008316560969, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.405671119689941, + "kd/policy_loss": -2.3955837726593017, + "kd/rkl_advantage_mean": 64.00839118957519, + "kd/rkl_advantage_p95": 84.91984851360321, + "kd/rkl_advantage_std": 36.31364707450072, + "kd/ssd_loss": 0.0, + "learning_rate": 6.343371435081456e-07, + "loss": -9.582335408528646, + "num_tokens": 125562057.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0260812362035117, + "sampling/importance_sampling_ratio/mean": 0.9998677372932434, + "sampling/importance_sampling_ratio/min": 0.9213335563739141, + "sampling/sampling_logp_difference/max": 0.11377484450737635, + "sampling/sampling_logp_difference/mean": 0.0016974368375182773, + "step": 9630, + "step_time": 4.45841189804875, + "student/entropy": 0.04545215287556251 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.0622222423553467, + "completions/mean_terminated_length": 1.0622222423553467, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04768144550422827, + "epoch": 0.36684008658337447, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.02642250061035, + "kd/policy_loss": -1.7763264179229736, + "kd/rkl_advantage_mean": 43.324218352635704, + "kd/rkl_advantage_p95": 56.260104179382324, + "kd/rkl_advantage_std": 23.13344126343727, + "kd/ssd_loss": 0.0, + "learning_rate": 6.331978885808681e-07, + "loss": -7.105304972330729, + "num_tokens": 125865537.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0421907742818197, + "sampling/importance_sampling_ratio/mean": 0.9999627153078715, + "sampling/importance_sampling_ratio/min": 0.918850201368332, + "sampling/sampling_logp_difference/max": 0.09539398173801601, + "sampling/sampling_logp_difference/mean": 0.0018466377485310658, + "step": 9660, + "step_time": 4.465730040559235, + "student/entropy": 0.04768144550422827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.733333333333333, + "completions/max_terminated_length": 7.733333333333333, + "completions/mean_length": 1.0791667064030965, + "completions/mean_terminated_length": 1.0791667064030965, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06356601429482302, + "epoch": 0.36797934151065204, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.752235412597656, + "kd/policy_loss": -3.7353809396425883, + "kd/rkl_advantage_mean": 71.40588607788087, + "kd/rkl_advantage_p95": 96.14847166538239, + "kd/rkl_advantage_std": 39.57485947410266, + "kd/ssd_loss": 0.0, + "learning_rate": 6.320586336535906e-07, + "loss": -14.941526285807292, + "num_tokens": 126165542.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0637479027112324, + "sampling/importance_sampling_ratio/mean": 1.0006835917631784, + "sampling/importance_sampling_ratio/min": 0.9291283925374348, + "sampling/sampling_logp_difference/max": 0.1268043768281738, + "sampling/sampling_logp_difference/mean": 0.0023383622814435513, + "step": 9690, + "step_time": 4.465573415493903, + "student/entropy": 0.06356601429482302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.766666666666667, + "completions/max_terminated_length": 6.766666666666667, + "completions/mean_length": 1.0641666928927103, + "completions/mean_terminated_length": 1.0641666928927103, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049460515721390645, + "epoch": 0.3691185964379296, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.546751976013184, + "kd/policy_loss": -1.6553228616714477, + "kd/rkl_advantage_mean": 53.87475218772888, + "kd/rkl_advantage_p95": 72.92127205530802, + "kd/rkl_advantage_std": 32.0483256260554, + "kd/ssd_loss": 0.0, + "learning_rate": 6.309193787263129e-07, + "loss": -6.6212921142578125, + "num_tokens": 126476141.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0496075948079426, + "sampling/importance_sampling_ratio/mean": 1.0002512673536936, + "sampling/importance_sampling_ratio/min": 0.9352504988511403, + "sampling/sampling_logp_difference/max": 0.08222940644870201, + "sampling/sampling_logp_difference/mean": 0.001565690707260122, + "step": 9720, + "step_time": 4.541031056475671, + "student/entropy": 0.049460515721390645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.3, + "completions/max_terminated_length": 7.3, + "completions/mean_length": 1.0722222566604613, + "completions/mean_terminated_length": 1.0722222566604613, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0528092997148633, + "epoch": 0.37025785136520717, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.119855880737305, + "kd/policy_loss": -3.3211581885814665, + "kd/rkl_advantage_mean": 69.81223824818929, + "kd/rkl_advantage_p95": 95.94466207822164, + "kd/rkl_advantage_std": 40.9964387734731, + "kd/ssd_loss": 0.0, + "learning_rate": 6.297801237990355e-07, + "loss": -13.284634399414063, + "num_tokens": 126780913.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0229911883672078, + "sampling/importance_sampling_ratio/mean": 1.0000972350438435, + "sampling/importance_sampling_ratio/min": 0.9580907801787059, + "sampling/sampling_logp_difference/max": 0.07670715605684866, + "sampling/sampling_logp_difference/mean": 0.001474857298308052, + "step": 9750, + "step_time": 4.348970888473559, + "student/entropy": 0.0528092997148633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.4, + "completions/max_terminated_length": 5.4, + "completions/mean_length": 1.0497222423553467, + "completions/mean_terminated_length": 1.0497222423553467, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04408194491018851, + "epoch": 0.37139710629248474, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 473.58966064453125, + "kd/policy_loss": -1.0175698320070903, + "kd/rkl_advantage_mean": 36.61836673418681, + "kd/rkl_advantage_p95": 50.67184909184774, + "kd/rkl_advantage_std": 22.052404448390007, + "kd/ssd_loss": 0.0, + "learning_rate": 6.286408688717579e-07, + "loss": -4.070280456542969, + "num_tokens": 127081068.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0185786843299867, + "sampling/importance_sampling_ratio/mean": 0.9998926818370819, + "sampling/importance_sampling_ratio/min": 0.9313314219315847, + "sampling/sampling_logp_difference/max": 0.07785288410571714, + "sampling/sampling_logp_difference/mean": 0.0013247126538772137, + "step": 9780, + "step_time": 4.2908302424786, + "student/entropy": 0.04408194491018851 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.533333333333333, + "completions/max_terminated_length": 7.533333333333333, + "completions/mean_length": 1.0741667032241822, + "completions/mean_terminated_length": 1.0741667032241822, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0580094238743186, + "epoch": 0.3725363612197623, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.838336944580078, + "kd/policy_loss": -2.890808816750844, + "kd/rkl_advantage_mean": 65.18028734525045, + "kd/rkl_advantage_p95": 90.29429137706757, + "kd/rkl_advantage_std": 38.36882737676303, + "kd/ssd_loss": 0.0, + "learning_rate": 6.275016139444803e-07, + "loss": -11.563235473632812, + "num_tokens": 127388159.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0285220821698506, + "sampling/importance_sampling_ratio/mean": 1.0002134064833323, + "sampling/importance_sampling_ratio/min": 0.9377275884151459, + "sampling/sampling_logp_difference/max": 0.09929902952474852, + "sampling/sampling_logp_difference/mean": 0.0017187340398474285, + "step": 9810, + "step_time": 4.2942552734438015, + "student/entropy": 0.0580094238743186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.233333333333333, + "completions/max_terminated_length": 7.233333333333333, + "completions/mean_length": 1.0772222518920898, + "completions/mean_terminated_length": 1.0772222518920898, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04247394236736, + "epoch": 0.3736756161470398, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.395169258117676, + "kd/policy_loss": -3.3017467816670734, + "kd/rkl_advantage_mean": 63.95693373680115, + "kd/rkl_advantage_p95": 84.90626000563303, + "kd/rkl_advantage_std": 34.54745748241742, + "kd/ssd_loss": 0.0, + "learning_rate": 6.263623590172027e-07, + "loss": -13.206988525390624, + "num_tokens": 127699357.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.028602127234141, + "sampling/importance_sampling_ratio/mean": 0.9998791793982188, + "sampling/importance_sampling_ratio/min": 0.9284165283044179, + "sampling/sampling_logp_difference/max": 0.08355816320205728, + "sampling/sampling_logp_difference/mean": 0.0014182941056787967, + "step": 9840, + "step_time": 4.389681235941437, + "student/entropy": 0.04247394236736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.9, + "completions/max_terminated_length": 6.9, + "completions/mean_length": 1.0675000230471292, + "completions/mean_terminated_length": 1.0675000230471292, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05183755376686652, + "epoch": 0.3748148710743174, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 94.53636169433594, + "kd/policy_loss": -2.547594984372457, + "kd/rkl_advantage_mean": 52.3553416411082, + "kd/rkl_advantage_p95": 67.89986741542816, + "kd/rkl_advantage_std": 27.045103122790653, + "kd/ssd_loss": 0.0, + "learning_rate": 6.252231040899252e-07, + "loss": -10.190380859375, + "num_tokens": 128002776.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0367724776268006, + "sampling/importance_sampling_ratio/mean": 1.0003674983978272, + "sampling/importance_sampling_ratio/min": 0.9532002389431, + "sampling/sampling_logp_difference/max": 0.08213533646582315, + "sampling/sampling_logp_difference/mean": 0.0015402039968951916, + "step": 9870, + "step_time": 4.372855866413253, + "student/entropy": 0.05183755376686652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.733333333333333, + "completions/max_terminated_length": 7.733333333333333, + "completions/mean_length": 1.071666689713796, + "completions/mean_terminated_length": 1.071666689713796, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048550058621913195, + "epoch": 0.37595412600159495, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.259170532226562, + "kd/policy_loss": -2.4745980461438495, + "kd/rkl_advantage_mean": 64.11798922220866, + "kd/rkl_advantage_p95": 84.97472310860952, + "kd/rkl_advantage_std": 35.975518396496774, + "kd/ssd_loss": 0.0, + "learning_rate": 6.240838491626475e-07, + "loss": -9.898391723632812, + "num_tokens": 128306322.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0228132764498392, + "sampling/importance_sampling_ratio/mean": 0.9995927929878234, + "sampling/importance_sampling_ratio/min": 0.8898847142855326, + "sampling/sampling_logp_difference/max": 0.10667507383041083, + "sampling/sampling_logp_difference/mean": 0.001663267573652168, + "step": 9900, + "step_time": 4.324538374513698, + "student/entropy": 0.048550058621913195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.0597222447395325, + "completions/mean_terminated_length": 1.0597222447395325, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.037148637945453325, + "epoch": 0.3770933809288725, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.012370109558105, + "kd/policy_loss": -2.0652756969134014, + "kd/rkl_advantage_mean": 46.66924597422282, + "kd/rkl_advantage_p95": 61.95771270593007, + "kd/rkl_advantage_std": 25.49369719326496, + "kd/ssd_loss": 0.0, + "learning_rate": 6.2294459423537e-07, + "loss": -8.261103312174479, + "num_tokens": 128610761.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0092241406440734, + "sampling/importance_sampling_ratio/mean": 0.9994898955027263, + "sampling/importance_sampling_ratio/min": 0.9271876712640127, + "sampling/sampling_logp_difference/max": 0.0864756710284079, + "sampling/sampling_logp_difference/mean": 0.0013935008222082008, + "step": 9930, + "step_time": 4.328308457683306, + "student/entropy": 0.037148637945453325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.081944477558136, + "completions/mean_terminated_length": 1.081944477558136, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.057698681578040124, + "epoch": 0.3782326358561501, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 28.287593841552734, + "kd/policy_loss": -3.7499740362167358, + "kd/rkl_advantage_mean": 56.58147389094035, + "kd/rkl_advantage_p95": 73.09139358997345, + "kd/rkl_advantage_std": 27.285162617762882, + "kd/ssd_loss": 0.0, + "learning_rate": 6.218053393080925e-07, + "loss": -14.999894205729166, + "num_tokens": 128919632.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0254425207773845, + "sampling/importance_sampling_ratio/mean": 0.9996902565161387, + "sampling/importance_sampling_ratio/min": 0.906793624162674, + "sampling/sampling_logp_difference/max": 0.11660261899232864, + "sampling/sampling_logp_difference/mean": 0.0019736276920108748, + "step": 9960, + "step_time": 4.292369778865638, + "student/entropy": 0.057698681578040124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.533333333333333, + "completions/max_terminated_length": 8.533333333333333, + "completions/mean_length": 1.102500041325887, + "completions/mean_terminated_length": 1.102500041325887, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04437075055514773, + "epoch": 0.37937189078342765, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 6.0519022941589355, + "kd/policy_loss": -6.044090898831685, + "kd/rkl_advantage_mean": 83.90727659861247, + "kd/rkl_advantage_p95": 107.38552770614623, + "kd/rkl_advantage_std": 40.1134229550759, + "kd/ssd_loss": 0.0, + "learning_rate": 6.206660843808149e-07, + "loss": -24.17636515299479, + "num_tokens": 129230809.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0459965030352274, + "sampling/importance_sampling_ratio/mean": 1.000034862756729, + "sampling/importance_sampling_ratio/min": 0.9302434623241425, + "sampling/sampling_logp_difference/max": 0.1048625921830535, + "sampling/sampling_logp_difference/mean": 0.001613439016606814, + "step": 9990, + "step_time": 4.264149659821609, + "student/entropy": 0.04437075055514773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.5, + "completions/max_terminated_length": 6.5, + "completions/mean_length": 1.0513889114061992, + "completions/mean_terminated_length": 1.0513889114061992, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.027810170718779167, + "epoch": 0.3805111457107052, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.870959281921387, + "kd/policy_loss": -1.278868850072225, + "kd/rkl_advantage_mean": 49.96694025993347, + "kd/rkl_advantage_p95": 67.7751563390096, + "kd/rkl_advantage_std": 30.231896049777667, + "kd/ssd_loss": 0.0, + "learning_rate": 6.195268294535374e-07, + "loss": -5.115475972493489, + "num_tokens": 129541506.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0770647565523783, + "sampling/importance_sampling_ratio/mean": 1.0002450545628865, + "sampling/importance_sampling_ratio/min": 0.9461902062098185, + "sampling/sampling_logp_difference/max": 0.10859687524692466, + "sampling/sampling_logp_difference/mean": 0.0013420156329326952, + "step": 10020, + "step_time": 4.355966415046714, + "student/entropy": 0.027810170718779167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.766666666666667, + "completions/max_terminated_length": 5.766666666666667, + "completions/mean_length": 1.0458333532015482, + "completions/mean_terminated_length": 1.0458333532015482, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03538784690511723, + "epoch": 0.3816504006379828, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.953463554382324, + "kd/policy_loss": -1.1652645389238994, + "kd/rkl_advantage_mean": 45.168710740407306, + "kd/rkl_advantage_p95": 62.0061052719752, + "kd/rkl_advantage_std": 28.008163531621296, + "kd/ssd_loss": 0.0, + "learning_rate": 6.183875745262598e-07, + "loss": -4.661060078938802, + "num_tokens": 129842311.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0102094928423564, + "sampling/importance_sampling_ratio/mean": 0.9998167852560679, + "sampling/importance_sampling_ratio/min": 0.9516620179017384, + "sampling/sampling_logp_difference/max": 0.04977647145278752, + "sampling/sampling_logp_difference/mean": 0.000853349374665413, + "step": 10050, + "step_time": 4.221206944928659, + "student/entropy": 0.03538784690511723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.433333333333334, + "completions/max_terminated_length": 7.433333333333334, + "completions/mean_length": 1.0619444688161215, + "completions/mean_terminated_length": 1.0619444688161215, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03993080537766218, + "epoch": 0.38278965556526034, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.00544261932373, + "kd/policy_loss": -2.7356419483820598, + "kd/rkl_advantage_mean": 64.10030358632406, + "kd/rkl_advantage_p95": 84.62214149634043, + "kd/rkl_advantage_std": 35.667018847664195, + "kd/ssd_loss": 0.0, + "learning_rate": 6.172483195989822e-07, + "loss": -10.942569986979167, + "num_tokens": 130141558.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0220232129096984, + "sampling/importance_sampling_ratio/mean": 0.999935797850291, + "sampling/importance_sampling_ratio/min": 0.950633688767751, + "sampling/sampling_logp_difference/max": 0.0929697809042409, + "sampling/sampling_logp_difference/mean": 0.0014797078004145684, + "step": 10080, + "step_time": 4.253610919613857, + "student/entropy": 0.03993080537766218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.133333333333334, + "completions/max_terminated_length": 7.133333333333334, + "completions/mean_length": 1.0647222479184468, + "completions/mean_terminated_length": 1.0647222479184468, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.050165833439677954, + "epoch": 0.38392891049253786, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 88.17749786376953, + "kd/policy_loss": -2.5626433690389, + "kd/rkl_advantage_mean": 56.257447703679404, + "kd/rkl_advantage_p95": 73.42151125272115, + "kd/rkl_advantage_std": 30.153357302149136, + "kd/ssd_loss": 0.0, + "learning_rate": 6.161090646717046e-07, + "loss": -10.25057373046875, + "num_tokens": 130451183.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0530882120132445, + "sampling/importance_sampling_ratio/mean": 1.0002315402030946, + "sampling/importance_sampling_ratio/min": 0.9553503672281901, + "sampling/sampling_logp_difference/max": 0.09102708099720379, + "sampling/sampling_logp_difference/mean": 0.00164232569707868, + "step": 10110, + "step_time": 4.295922764181159, + "student/entropy": 0.050165833439677954 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.666666666666667, + "completions/max_terminated_length": 6.666666666666667, + "completions/mean_length": 1.065833361943563, + "completions/mean_terminated_length": 1.065833361943563, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04349759708469113, + "epoch": 0.3850681654198154, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.051690101623535, + "kd/policy_loss": -1.7649108469486237, + "kd/rkl_advantage_mean": 52.945310322443646, + "kd/rkl_advantage_p95": 73.2467985868454, + "kd/rkl_advantage_std": 32.15300884445508, + "kd/ssd_loss": 0.0, + "learning_rate": 6.149698097444272e-07, + "loss": -7.059642537434896, + "num_tokens": 130760148.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0720045765240986, + "sampling/importance_sampling_ratio/mean": 1.000155089298884, + "sampling/importance_sampling_ratio/min": 0.9385699133078257, + "sampling/sampling_logp_difference/max": 0.10159194103131691, + "sampling/sampling_logp_difference/mean": 0.001554353853862267, + "step": 10140, + "step_time": 4.302429276879411, + "student/entropy": 0.04349759708469113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.0700000246365866, + "completions/mean_terminated_length": 1.0700000246365866, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05718285307909052, + "epoch": 0.386207420347093, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.289457321166992, + "kd/policy_loss": -1.9685159921646118, + "kd/rkl_advantage_mean": 46.735379950205484, + "kd/rkl_advantage_p95": 62.03057126998901, + "kd/rkl_advantage_std": 25.873121647040048, + "kd/ssd_loss": 0.0, + "learning_rate": 6.138305548171496e-07, + "loss": -7.874065653483073, + "num_tokens": 131069568.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.004285195469856262, + "sampling/importance_sampling_ratio/max": 1.0108763933181764, + "sampling/importance_sampling_ratio/mean": 1.0000134229660034, + "sampling/importance_sampling_ratio/min": 0.9508651594320933, + "sampling/sampling_logp_difference/max": 0.05623992298108836, + "sampling/sampling_logp_difference/mean": 0.0012711818184470759, + "step": 10170, + "step_time": 4.367414963354046, + "student/entropy": 0.05718285307909052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.0, + "completions/max_terminated_length": 9.0, + "completions/mean_length": 1.1013889233271281, + "completions/mean_terminated_length": 1.1013889233271281, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05343762616006036, + "epoch": 0.38734667527437056, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 32.489288330078125, + "kd/policy_loss": -6.059292256832123, + "kd/rkl_advantage_mean": 84.66604784329732, + "kd/rkl_advantage_p95": 107.35172475973765, + "kd/rkl_advantage_std": 39.41490786671638, + "kd/ssd_loss": 0.0, + "learning_rate": 6.12691299889872e-07, + "loss": -24.237166341145834, + "num_tokens": 131375957.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0645221630732218, + "sampling/importance_sampling_ratio/mean": 0.9999563018480937, + "sampling/importance_sampling_ratio/min": 0.8978176474571228, + "sampling/sampling_logp_difference/max": 0.12846089291075866, + "sampling/sampling_logp_difference/mean": 0.0020727490370821516, + "step": 10200, + "step_time": 4.218270132290976, + "student/entropy": 0.05343762616006036 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.05805557568868, + "completions/mean_terminated_length": 1.05805557568868, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.038703535016005236, + "epoch": 0.3884859302016481, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.867291450500488, + "kd/policy_loss": -1.5629056294759114, + "kd/rkl_advantage_mean": 46.2660928885142, + "kd/rkl_advantage_p95": 62.191306408246355, + "kd/rkl_advantage_std": 26.8408834785223, + "kd/ssd_loss": 0.0, + "learning_rate": 6.115520449625944e-07, + "loss": -6.2516230265299475, + "num_tokens": 131684470.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0062782168388367, + "sampling/importance_sampling_ratio/mean": 0.9995807905991873, + "sampling/importance_sampling_ratio/min": 0.9257072071234386, + "sampling/sampling_logp_difference/max": 0.08283765108014146, + "sampling/sampling_logp_difference/mean": 0.001132313656853512, + "step": 10230, + "step_time": 4.308224556761949, + "student/entropy": 0.038703535016005236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.033333333333333, + "completions/max_terminated_length": 7.033333333333333, + "completions/mean_length": 1.0563889185587565, + "completions/mean_terminated_length": 1.0563889185587565, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048775643948465584, + "epoch": 0.3896251851289257, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.018552780151367, + "kd/policy_loss": -2.636602326234182, + "kd/rkl_advantage_mean": 64.56550165812175, + "kd/rkl_advantage_p95": 89.76185932954152, + "kd/rkl_advantage_std": 39.01474737127622, + "kd/ssd_loss": 0.0, + "learning_rate": 6.104127900353169e-07, + "loss": -10.546410115559896, + "num_tokens": 132002649.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0323549111684163, + "sampling/importance_sampling_ratio/mean": 1.00023965438207, + "sampling/importance_sampling_ratio/min": 0.9674233416716258, + "sampling/sampling_logp_difference/max": 0.06545505017663042, + "sampling/sampling_logp_difference/mean": 0.0011634811710488672, + "step": 10260, + "step_time": 4.314920138889769, + "student/entropy": 0.048775643948465584 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.733333333333333, + "completions/max_terminated_length": 6.733333333333333, + "completions/mean_length": 1.0702778021494548, + "completions/mean_terminated_length": 1.0702778021494548, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04238721973573168, + "epoch": 0.39076444005620325, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.945466995239258, + "kd/policy_loss": -2.1666825373967487, + "kd/rkl_advantage_mean": 54.73877582550049, + "kd/rkl_advantage_p95": 73.42450866699218, + "kd/rkl_advantage_std": 31.013311371207237, + "kd/ssd_loss": 0.0, + "learning_rate": 6.092735351080393e-07, + "loss": -8.666731770833334, + "num_tokens": 132312374.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0269407788912455, + "sampling/importance_sampling_ratio/mean": 0.9998952190081278, + "sampling/importance_sampling_ratio/min": 0.9475167870521546, + "sampling/sampling_logp_difference/max": 0.0792722770323356, + "sampling/sampling_logp_difference/mean": 0.0012994511140277608, + "step": 10290, + "step_time": 4.3193098695366645, + "student/entropy": 0.04238721973573168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.866666666666666, + "completions/max_terminated_length": 5.866666666666666, + "completions/mean_length": 1.0483333547910054, + "completions/mean_terminated_length": 1.0483333547910054, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.033271205890923736, + "epoch": 0.3919036949834808, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.37045669555664, + "kd/policy_loss": -0.47443214257558186, + "kd/rkl_advantage_mean": 43.62829195658366, + "kd/rkl_advantage_p95": 61.57399093310038, + "kd/rkl_advantage_std": 29.392284807562827, + "kd/ssd_loss": 0.0, + "learning_rate": 6.081342801807617e-07, + "loss": -1.8977315266927084, + "num_tokens": 132617308.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0202199419339497, + "sampling/importance_sampling_ratio/mean": 0.9998335758845012, + "sampling/importance_sampling_ratio/min": 0.942125607530276, + "sampling/sampling_logp_difference/max": 0.07990852013075103, + "sampling/sampling_logp_difference/mean": 0.0010399512471243118, + "step": 10320, + "step_time": 4.305071547844758, + "student/entropy": 0.033271205890923736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.366666666666666, + "completions/max_terminated_length": 7.366666666666666, + "completions/mean_length": 1.0886111458142598, + "completions/mean_terminated_length": 1.0886111458142598, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05611841998373469, + "epoch": 0.3930429499107584, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.302277565002441, + "kd/policy_loss": -3.5136493941148124, + "kd/rkl_advantage_mean": 64.42968519528706, + "kd/rkl_advantage_p95": 84.64493745962778, + "kd/rkl_advantage_std": 33.7767775674661, + "kd/ssd_loss": 0.0, + "learning_rate": 6.069950252534843e-07, + "loss": -14.054597981770833, + "num_tokens": 132918555.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0597009142239888, + "sampling/importance_sampling_ratio/mean": 1.000137448310852, + "sampling/importance_sampling_ratio/min": 0.9272212545077007, + "sampling/sampling_logp_difference/max": 0.10112287563582262, + "sampling/sampling_logp_difference/mean": 0.0018297831721914312, + "step": 10350, + "step_time": 4.333443595220645, + "student/entropy": 0.05611841998373469 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.7, + "completions/max_terminated_length": 6.7, + "completions/mean_length": 1.0677778005599976, + "completions/mean_terminated_length": 1.0677778005599976, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.041351465663562216, + "epoch": 0.3941822048380359, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.01654052734375, + "kd/policy_loss": -2.2311511675516766, + "kd/rkl_advantage_mean": 55.49196716944377, + "kd/rkl_advantage_p95": 73.33408404191336, + "kd/rkl_advantage_std": 31.282723458111285, + "kd/ssd_loss": 0.0, + "learning_rate": 6.058557703262066e-07, + "loss": -8.924606323242188, + "num_tokens": 133223991.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.035687788327535, + "sampling/importance_sampling_ratio/mean": 1.000114417076111, + "sampling/importance_sampling_ratio/min": 0.9518165747324626, + "sampling/sampling_logp_difference/max": 0.0783600082465758, + "sampling/sampling_logp_difference/mean": 0.0011919254267316623, + "step": 10380, + "step_time": 4.283620600084153, + "student/entropy": 0.041351465663562216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.0630555788675944, + "completions/mean_terminated_length": 1.0630555788675944, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.046509655720243855, + "epoch": 0.39532145976531347, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.637419700622559, + "kd/policy_loss": -1.61842170159022, + "kd/rkl_advantage_mean": 54.140386994679766, + "kd/rkl_advantage_p95": 73.59273122151693, + "kd/rkl_advantage_std": 32.14762073854605, + "kd/ssd_loss": 0.0, + "learning_rate": 6.047165153989291e-07, + "loss": -6.4736887613932295, + "num_tokens": 133534378.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0181899944941202, + "sampling/importance_sampling_ratio/mean": 0.9998906294504801, + "sampling/importance_sampling_ratio/min": 0.9500101407368978, + "sampling/sampling_logp_difference/max": 0.0653124158270657, + "sampling/sampling_logp_difference/mean": 0.0010622938735953844, + "step": 10410, + "step_time": 4.406731648331818, + "student/entropy": 0.046509655720243855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.466666666666667, + "completions/max_terminated_length": 7.466666666666667, + "completions/mean_length": 1.0858333667119344, + "completions/mean_terminated_length": 1.0858333667119344, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0533822063356638, + "epoch": 0.39646071469259103, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.073400497436523, + "kd/policy_loss": -4.236212166150411, + "kd/rkl_advantage_mean": 62.1953884601593, + "kd/rkl_advantage_p95": 79.25384708245595, + "kd/rkl_advantage_std": 29.232659675677617, + "kd/ssd_loss": 0.0, + "learning_rate": 6.035772604716515e-07, + "loss": -16.94485066731771, + "num_tokens": 133849591.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0215216596921286, + "sampling/importance_sampling_ratio/mean": 0.9997353752454122, + "sampling/importance_sampling_ratio/min": 0.9460016985734304, + "sampling/sampling_logp_difference/max": 0.06374582997523248, + "sampling/sampling_logp_difference/mean": 0.001429751438748402, + "step": 10440, + "step_time": 4.392644595960155, + "student/entropy": 0.0533822063356638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.05583336353302, + "completions/mean_terminated_length": 1.05583336353302, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04367889647061626, + "epoch": 0.3975999696198686, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.891188621520996, + "kd/policy_loss": -2.278918619950612, + "kd/rkl_advantage_mean": 51.10216215451558, + "kd/rkl_advantage_p95": 73.13406165440877, + "kd/rkl_advantage_std": 31.314032846689223, + "kd/ssd_loss": 0.0, + "learning_rate": 6.024380055443739e-07, + "loss": -9.115673828125, + "num_tokens": 134167560.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.040869907538096, + "sampling/importance_sampling_ratio/mean": 1.00007763504982, + "sampling/importance_sampling_ratio/min": 0.9336876153945923, + "sampling/sampling_logp_difference/max": 0.09347009250583747, + "sampling/sampling_logp_difference/mean": 0.0014384308082905288, + "step": 10470, + "step_time": 4.457021307712421, + "student/entropy": 0.04367889647061626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0691666920979819, + "completions/mean_terminated_length": 1.0691666920979819, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04057673690840602, + "epoch": 0.39873922454714616, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 46.48445510864258, + "kd/policy_loss": -2.6125645279884337, + "kd/rkl_advantage_mean": 59.23271369934082, + "kd/rkl_advantage_p95": 78.76204336484274, + "kd/rkl_advantage_std": 32.9168572584788, + "kd/ssd_loss": 0.0, + "learning_rate": 6.012987506170963e-07, + "loss": -10.450259399414062, + "num_tokens": 134473313.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0054235895474752, + "sampling/importance_sampling_ratio/mean": 0.9996112147967021, + "sampling/importance_sampling_ratio/min": 0.9351443072160085, + "sampling/sampling_logp_difference/max": 0.07449241305390994, + "sampling/sampling_logp_difference/mean": 0.0011928155242154996, + "step": 10500, + "step_time": 4.353720004340478, + "student/entropy": 0.04057673690840602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.366666666666666, + "completions/max_terminated_length": 6.366666666666666, + "completions/mean_length": 1.0541666865348815, + "completions/mean_terminated_length": 1.0541666865348815, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0516138452415665, + "epoch": 0.39987847947442373, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 26.38564109802246, + "kd/policy_loss": -0.5987134496370952, + "kd/rkl_advantage_mean": 48.445869032541914, + "kd/rkl_advantage_p95": 67.98616420427958, + "kd/rkl_advantage_std": 31.992851788302264, + "kd/ssd_loss": 0.0, + "learning_rate": 6.001594956898188e-07, + "loss": -2.3948542277018228, + "num_tokens": 134798364.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0300590554873148, + "sampling/importance_sampling_ratio/mean": 0.999933926264445, + "sampling/importance_sampling_ratio/min": 0.9233426064252853, + "sampling/sampling_logp_difference/max": 0.08222977230325342, + "sampling/sampling_logp_difference/mean": 0.001211774216305154, + "step": 10530, + "step_time": 4.492175256068974, + "student/entropy": 0.0516138452415665 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 23.8, + "completions/max_terminated_length": 7.033333333333333, + "completions/mean_length": 1.2033333698908488, + "completions/mean_terminated_length": 1.0614075899124145, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05410677067314585, + "epoch": 0.4010177344017013, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 414.1174621582031, + "kd/policy_loss": -2.640220445394516, + "kd/rkl_advantage_mean": 64.49839502970377, + "kd/rkl_advantage_p95": 84.59516274929047, + "kd/rkl_advantage_std": 34.59711539149284, + "kd/ssd_loss": 0.0, + "learning_rate": 5.990202407625414e-07, + "loss": -10.560880533854167, + "num_tokens": 135112464.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0558635592460632, + "sampling/importance_sampling_ratio/mean": 1.0003626823425293, + "sampling/importance_sampling_ratio/min": 0.953032499551773, + "sampling/sampling_logp_difference/max": 0.08940341640263796, + "sampling/sampling_logp_difference/mean": 0.0013224030709049353, + "step": 10560, + "step_time": 4.463589711421324, + "student/entropy": 0.05410677067314585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 21.533333333333335, + "completions/max_terminated_length": 4.5, + "completions/mean_length": 1.1811111410458883, + "completions/mean_terminated_length": 1.0391666849454244, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.051080021044860285, + "epoch": 0.40215698932897886, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.937267303466797, + "kd/policy_loss": -0.19445254206657409, + "kd/rkl_advantage_mean": 32.86770774523417, + "kd/rkl_advantage_p95": 44.81240456104278, + "kd/rkl_advantage_std": 19.661983728408813, + "kd/ssd_loss": 0.0, + "learning_rate": 5.978809858352637e-07, + "loss": -0.7778103510538737, + "num_tokens": 135424364.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0413977344830831, + "sampling/importance_sampling_ratio/mean": 1.0000808815161386, + "sampling/importance_sampling_ratio/min": 0.9378605971733729, + "sampling/sampling_logp_difference/max": 0.13052070164121687, + "sampling/sampling_logp_difference/mean": 0.0017317969429617127, + "step": 10590, + "step_time": 4.449399976544858, + "student/entropy": 0.051080021044860285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 24.3, + "completions/max_terminated_length": 7.6, + "completions/mean_length": 1.2516667048136394, + "completions/mean_terminated_length": 1.1097689429918924, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07189544811844825, + "epoch": 0.40329624425625643, + "eval/gt_acc_batch": 0.0025000001614292463, + "frac_reward_zero_std": 1.0, + "grad_norm": 29.54489517211914, + "kd/policy_loss": -4.551234009861946, + "kd/rkl_advantage_mean": 66.22717108726502, + "kd/rkl_advantage_p95": 84.83763949076335, + "kd/rkl_advantage_std": 29.80577287375927, + "kd/ssd_loss": 0.0, + "learning_rate": 5.967417309079862e-07, + "loss": -18.20494181315104, + "num_tokens": 135740030.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000000993410745, + "rewards/gt_logging_reward/std": 0.021682794392108917, + "sampling/importance_sampling_ratio/max": 1.0773635069529215, + "sampling/importance_sampling_ratio/mean": 1.0000378807385764, + "sampling/importance_sampling_ratio/min": 0.8801457663377126, + "sampling/sampling_logp_difference/max": 0.13938520528996984, + "sampling/sampling_logp_difference/mean": 0.002471837229677476, + "step": 10620, + "step_time": 4.414635608678994, + "student/entropy": 0.07189544811844825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.033333333333333, + "completions/max_terminated_length": 9.033333333333333, + "completions/mean_length": 1.1380555947621664, + "completions/mean_terminated_length": 1.1380555947621664, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.08347487924620509, + "epoch": 0.40443549918353394, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 59.495670318603516, + "kd/policy_loss": -7.7463063955307, + "kd/rkl_advantage_mean": 86.5432482878367, + "kd/rkl_advantage_p95": 107.26563267707824, + "kd/rkl_advantage_std": 35.595209221045174, + "kd/ssd_loss": 0.0, + "learning_rate": 5.956024759807086e-07, + "loss": -30.985227457682292, + "num_tokens": 136032951.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.1155561486879984, + "sampling/importance_sampling_ratio/mean": 1.0003206352392833, + "sampling/importance_sampling_ratio/min": 0.8919048031171163, + "sampling/sampling_logp_difference/max": 0.15989847363283236, + "sampling/sampling_logp_difference/mean": 0.003201785741839558, + "step": 10650, + "step_time": 4.392758996163805, + "student/entropy": 0.08347487924620509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.433333333333334, + "completions/max_terminated_length": 7.433333333333334, + "completions/mean_length": 1.0800000190734864, + "completions/mean_terminated_length": 1.0800000190734864, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04871978216494123, + "epoch": 0.4055747541108115, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.903646469116211, + "kd/policy_loss": -3.3265979448954264, + "kd/rkl_advantage_mean": 57.743504190444945, + "kd/rkl_advantage_p95": 73.34396850268045, + "kd/rkl_advantage_std": 28.350233965118726, + "kd/ssd_loss": 0.0, + "learning_rate": 5.94463221053431e-07, + "loss": -13.306390380859375, + "num_tokens": 136338791.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0852608879407246, + "sampling/importance_sampling_ratio/mean": 1.0005278746287027, + "sampling/importance_sampling_ratio/min": 0.9407489796479543, + "sampling/sampling_logp_difference/max": 0.08425431759096683, + "sampling/sampling_logp_difference/mean": 0.0016404484694551988, + "step": 10680, + "step_time": 4.351236722820128, + "student/entropy": 0.04871978216494123 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.070555579662323, + "completions/mean_terminated_length": 1.070555579662323, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06143165767813722, + "epoch": 0.4067140090380891, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.625713348388672, + "kd/policy_loss": -2.0249652306238812, + "kd/rkl_advantage_mean": 55.12375726699829, + "kd/rkl_advantage_p95": 73.70309486389161, + "kd/rkl_advantage_std": 31.574395288030306, + "kd/ssd_loss": 0.0, + "learning_rate": 5.933239661261534e-07, + "loss": -8.099859619140625, + "num_tokens": 136645845.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0179280559221904, + "sampling/importance_sampling_ratio/mean": 0.999941756327947, + "sampling/importance_sampling_ratio/min": 0.9322610666354497, + "sampling/sampling_logp_difference/max": 0.08189455076741675, + "sampling/sampling_logp_difference/mean": 0.0015075254797314605, + "step": 10710, + "step_time": 4.389390837587416, + "student/entropy": 0.06143165767813722 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.896551724137931, + "completions/max_terminated_length": 6.896551724137931, + "completions/mean_length": 1.077586239781873, + "completions/mean_terminated_length": 1.077586239781873, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05615162692064869, + "epoch": 0.40785326396536664, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.97662353515625, + "kd/policy_loss": -4.071750718971779, + "kd/rkl_advantage_mean": 59.20172438128241, + "kd/rkl_advantage_p95": 76.09593487608022, + "kd/rkl_advantage_std": 28.36062195383269, + "kd/ssd_loss": 0.0, + "learning_rate": 5.92184711198876e-07, + "loss": -15.744102986653646, + "num_tokens": 136946699.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0294726922594268, + "sampling/importance_sampling_ratio/mean": 1.0000316665090363, + "sampling/importance_sampling_ratio/min": 0.9313817702490708, + "sampling/sampling_logp_difference/max": 0.09616284713084841, + "sampling/sampling_logp_difference/mean": 0.0015474716351961653, + "step": 10740, + "step_time": 4.3030477597184165, + "student/entropy": 0.05615162692064869 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.1, + "completions/max_terminated_length": 7.1, + "completions/mean_length": 1.0777778108914693, + "completions/mean_terminated_length": 1.0777778108914693, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04739198035870989, + "epoch": 0.4089925188926442, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.184640884399414, + "kd/policy_loss": -3.5077362934748333, + "kd/rkl_advantage_mean": 61.05262983640035, + "kd/rkl_advantage_p95": 79.07808237870535, + "kd/rkl_advantage_std": 30.980360075831413, + "kd/ssd_loss": 0.0, + "learning_rate": 5.910454562715983e-07, + "loss": -14.030946858723958, + "num_tokens": 137244339.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.022573971748352, + "sampling/importance_sampling_ratio/mean": 0.9995502849419912, + "sampling/importance_sampling_ratio/min": 0.9116539597511292, + "sampling/sampling_logp_difference/max": 0.08918819470951954, + "sampling/sampling_logp_difference/mean": 0.001725875802609759, + "step": 10770, + "step_time": 4.381624625522333, + "student/entropy": 0.04739198035870989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.733333333333333, + "completions/max_terminated_length": 6.733333333333333, + "completions/mean_length": 1.0505555788675944, + "completions/mean_terminated_length": 1.0505555788675944, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.039566436177119614, + "epoch": 0.4101317738199218, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.99624252319336, + "kd/policy_loss": -1.3380534887313842, + "kd/rkl_advantage_mean": 53.376574754714966, + "kd/rkl_advantage_p95": 73.42894324461619, + "kd/rkl_advantage_std": 32.9504029194514, + "kd/ssd_loss": 0.0, + "learning_rate": 5.899062013443208e-07, + "loss": -5.352212524414062, + "num_tokens": 137553913.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0083179275194805, + "sampling/importance_sampling_ratio/mean": 0.999737028280894, + "sampling/importance_sampling_ratio/min": 0.9304774562517802, + "sampling/sampling_logp_difference/max": 0.08369868979789316, + "sampling/sampling_logp_difference/mean": 0.0011833456490421667, + "step": 10800, + "step_time": 4.386044740847622, + "student/entropy": 0.039566436177119614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.666666666666667, + "completions/max_terminated_length": 5.666666666666667, + "completions/mean_length": 1.0494444648424783, + "completions/mean_terminated_length": 1.0494444648424783, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.040370258906235294, + "epoch": 0.41127102874719934, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.846839904785156, + "kd/policy_loss": -0.7860488891601562, + "kd/rkl_advantage_mean": 40.79295622507731, + "kd/rkl_advantage_p95": 56.429181631406145, + "kd/rkl_advantage_std": 25.474414284030598, + "kd/ssd_loss": 0.0, + "learning_rate": 5.887669464170433e-07, + "loss": -3.144195302327474, + "num_tokens": 137863507.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0039905548095702, + "sampling/importance_sampling_ratio/mean": 0.9997895340124766, + "sampling/importance_sampling_ratio/min": 0.9431172430515289, + "sampling/sampling_logp_difference/max": 0.058742870949208735, + "sampling/sampling_logp_difference/mean": 0.0009572283587961768, + "step": 10830, + "step_time": 4.354024590004701, + "student/entropy": 0.040370258906235294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.0772222479184468, + "completions/mean_terminated_length": 1.0772222479184468, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.059415012163420516, + "epoch": 0.4124102836744769, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.873374938964844, + "kd/policy_loss": -2.9004260381062825, + "kd/rkl_advantage_mean": 63.21875813802083, + "kd/rkl_advantage_p95": 84.43258035977682, + "kd/rkl_advantage_std": 35.14891779124737, + "kd/ssd_loss": 0.0, + "learning_rate": 5.876276914897656e-07, + "loss": -11.601702880859374, + "num_tokens": 138163905.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0339414914449057, + "sampling/importance_sampling_ratio/mean": 1.0000114222367604, + "sampling/importance_sampling_ratio/min": 0.9083704153696696, + "sampling/sampling_logp_difference/max": 0.1080609112046659, + "sampling/sampling_logp_difference/mean": 0.0016896362620173022, + "step": 10860, + "step_time": 4.355564656322046, + "student/entropy": 0.059415012163420516 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.766666666666667, + "completions/max_terminated_length": 7.766666666666667, + "completions/mean_length": 1.0813889225323996, + "completions/mean_terminated_length": 1.0813889225323996, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04646703926846385, + "epoch": 0.4135495386017545, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.193315505981445, + "kd/policy_loss": -3.967756430308024, + "kd/rkl_advantage_mean": 69.46191207567851, + "kd/rkl_advantage_p95": 90.3965636809667, + "kd/rkl_advantage_std": 36.082299933830896, + "kd/ssd_loss": 0.0, + "learning_rate": 5.864884365624881e-07, + "loss": -15.871026611328125, + "num_tokens": 138476406.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.047549851735433, + "sampling/importance_sampling_ratio/mean": 1.0002282361189525, + "sampling/importance_sampling_ratio/min": 0.952148582537969, + "sampling/sampling_logp_difference/max": 0.08765391269698738, + "sampling/sampling_logp_difference/mean": 0.0014086939248954878, + "step": 10890, + "step_time": 4.306191521766595, + "student/entropy": 0.04646703926846385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.063611133893331, + "completions/mean_terminated_length": 1.063611133893331, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0415394120849669, + "epoch": 0.41468879352903204, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.572956085205078, + "kd/policy_loss": -0.7590126077334086, + "kd/rkl_advantage_mean": 44.28872855504354, + "kd/rkl_advantage_p95": 61.89448608557383, + "kd/rkl_advantage_std": 28.47199950714906, + "kd/ssd_loss": 0.0, + "learning_rate": 5.853491816352105e-07, + "loss": -3.036049397786458, + "num_tokens": 138795459.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.013125971953074, + "sampling/importance_sampling_ratio/mean": 0.9996853927771251, + "sampling/importance_sampling_ratio/min": 0.9275313079357147, + "sampling/sampling_logp_difference/max": 0.07162740354736646, + "sampling/sampling_logp_difference/mean": 0.0012225378935302917, + "step": 10920, + "step_time": 4.431671973783523, + "student/entropy": 0.0415394120849669 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.0655555804570516, + "completions/mean_terminated_length": 1.0655555804570516, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05614906034121911, + "epoch": 0.41582804845630955, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.734956741333008, + "kd/policy_loss": -2.113850518067678, + "kd/rkl_advantage_mean": 50.52693486213684, + "kd/rkl_advantage_p95": 67.85159420172373, + "kd/rkl_advantage_std": 28.317848418156306, + "kd/ssd_loss": 0.0, + "learning_rate": 5.84209926707933e-07, + "loss": -8.455403645833334, + "num_tokens": 139095727.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.013589910666148, + "sampling/importance_sampling_ratio/mean": 1.0003472248713174, + "sampling/importance_sampling_ratio/min": 0.9667099098364512, + "sampling/sampling_logp_difference/max": 0.04701155570025246, + "sampling/sampling_logp_difference/mean": 0.00111487601728489, + "step": 10950, + "step_time": 4.3567275868612345, + "student/entropy": 0.05614906034121911 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.333333333333334, + "completions/max_terminated_length": 8.333333333333334, + "completions/mean_length": 1.0875000357627869, + "completions/mean_terminated_length": 1.0875000357627869, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05196800713116924, + "epoch": 0.4169673033835871, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 31.596134185791016, + "kd/policy_loss": -4.370733082294464, + "kd/rkl_advantage_mean": 76.7651653766632, + "kd/rkl_advantage_p95": 101.79664885203043, + "kd/rkl_advantage_std": 40.51910613675912, + "kd/ssd_loss": 0.0, + "learning_rate": 5.830706717806554e-07, + "loss": -17.48293253580729, + "num_tokens": 139416306.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0152411540349324, + "sampling/importance_sampling_ratio/mean": 0.999592270453771, + "sampling/importance_sampling_ratio/min": 0.8981493224700292, + "sampling/sampling_logp_difference/max": 0.11744827661508074, + "sampling/sampling_logp_difference/mean": 0.0017854888554817685, + "step": 10980, + "step_time": 4.569594681895493, + "student/entropy": 0.05196800713116924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.633333333333334, + "completions/max_terminated_length": 6.633333333333334, + "completions/mean_length": 1.076388915379842, + "completions/mean_terminated_length": 1.076388915379842, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05012108534574509, + "epoch": 0.4181065583108647, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.353656768798828, + "kd/policy_loss": -2.9508484105269113, + "kd/rkl_advantage_mean": 55.14403033256531, + "kd/rkl_advantage_p95": 73.35032591025035, + "kd/rkl_advantage_std": 29.75554750363032, + "kd/ssd_loss": 0.0, + "learning_rate": 5.819314168533779e-07, + "loss": -11.803392537434895, + "num_tokens": 139717293.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0987240235010782, + "sampling/importance_sampling_ratio/mean": 1.0008368790149689, + "sampling/importance_sampling_ratio/min": 0.9497633179028829, + "sampling/sampling_logp_difference/max": 0.12306764487487575, + "sampling/sampling_logp_difference/mean": 0.0018574070369747158, + "step": 11010, + "step_time": 4.335785773590517, + "student/entropy": 0.05012108534574509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.0713889201482136, + "completions/mean_terminated_length": 1.0713889201482136, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049709356296807526, + "epoch": 0.41924581323814225, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.44926643371582, + "kd/policy_loss": -3.0885506669680276, + "kd/rkl_advantage_mean": 51.71510140101115, + "kd/rkl_advantage_p95": 67.35323789119721, + "kd/rkl_advantage_std": 26.45073095858097, + "kd/ssd_loss": 0.0, + "learning_rate": 5.807921619261002e-07, + "loss": -12.354203287760416, + "num_tokens": 140021934.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0205790638923644, + "sampling/importance_sampling_ratio/mean": 0.9998487552007039, + "sampling/importance_sampling_ratio/min": 0.9241643091042836, + "sampling/sampling_logp_difference/max": 0.07810228583402931, + "sampling/sampling_logp_difference/mean": 0.0013707170030102134, + "step": 11040, + "step_time": 4.390761039569043, + "student/entropy": 0.049709356296807526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.933333333333334, + "completions/max_terminated_length": 6.933333333333334, + "completions/mean_length": 1.0791667024294536, + "completions/mean_terminated_length": 1.0791667024294536, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.050915813508133095, + "epoch": 0.4203850681654198, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.53714656829834, + "kd/policy_loss": -3.2353841106096906, + "kd/rkl_advantage_mean": 63.2017012278239, + "kd/rkl_advantage_p95": 84.63360268274943, + "kd/rkl_advantage_std": 35.00664997796218, + "kd/ssd_loss": 0.0, + "learning_rate": 5.796529069988227e-07, + "loss": -12.941536458333333, + "num_tokens": 140334227.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0481099923451742, + "sampling/importance_sampling_ratio/mean": 0.9999427636464436, + "sampling/importance_sampling_ratio/min": 0.9302429219086965, + "sampling/sampling_logp_difference/max": 0.11217011596697073, + "sampling/sampling_logp_difference/mean": 0.0019465202688782786, + "step": 11070, + "step_time": 4.348901417606976, + "student/entropy": 0.050915813508133095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.266666666666667, + "completions/max_terminated_length": 7.266666666666667, + "completions/mean_length": 1.070555583635966, + "completions/mean_terminated_length": 1.070555583635966, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04695388460531831, + "epoch": 0.4215243230926974, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 49.35267639160156, + "kd/policy_loss": -2.314220408598582, + "kd/rkl_advantage_mean": 59.10985008875529, + "kd/rkl_advantage_p95": 78.84839284420013, + "kd/rkl_advantage_std": 33.56372203081846, + "kd/ssd_loss": 0.0, + "learning_rate": 5.785136520715452e-07, + "loss": -9.256883748372395, + "num_tokens": 140649345.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0224343140920003, + "sampling/importance_sampling_ratio/mean": 0.9998160799344381, + "sampling/importance_sampling_ratio/min": 0.9232208395997683, + "sampling/sampling_logp_difference/max": 0.11885266590397806, + "sampling/sampling_logp_difference/mean": 0.0014549850340699777, + "step": 11100, + "step_time": 4.427173984822972, + "student/entropy": 0.04695388460531831 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 23.666666666666668, + "completions/max_terminated_length": 6.9, + "completions/mean_length": 1.2188889304796855, + "completions/mean_terminated_length": 1.0769631505012511, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05794842649872104, + "epoch": 0.42266357801997495, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.673370361328125, + "kd/policy_loss": -3.4071746448675793, + "kd/rkl_advantage_mean": 61.858478450775145, + "kd/rkl_advantage_p95": 79.27649715741475, + "kd/rkl_advantage_std": 29.918202595909438, + "kd/ssd_loss": 0.0, + "learning_rate": 5.773743971442677e-07, + "loss": -13.628702799479166, + "num_tokens": 140966309.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0666353424390158, + "sampling/importance_sampling_ratio/mean": 1.0003898421923318, + "sampling/importance_sampling_ratio/min": 0.9282915651798248, + "sampling/sampling_logp_difference/max": 0.09681506621806572, + "sampling/sampling_logp_difference/mean": 0.0016889701781716818, + "step": 11130, + "step_time": 4.512850998528302, + "student/entropy": 0.05794842649872104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.0683333595593771, + "completions/mean_terminated_length": 1.0683333595593771, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04329327641365429, + "epoch": 0.4238028329472525, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.54064178466797, + "kd/policy_loss": -2.873375153541565, + "kd/rkl_advantage_mean": 55.39872854550679, + "kd/rkl_advantage_p95": 73.39099794228872, + "kd/rkl_advantage_std": 29.888448438048364, + "kd/ssd_loss": 0.0, + "learning_rate": 5.762351422169901e-07, + "loss": -11.493500773111979, + "num_tokens": 141278699.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0467960198720296, + "sampling/importance_sampling_ratio/mean": 1.000177017847697, + "sampling/importance_sampling_ratio/min": 0.9492423931757609, + "sampling/sampling_logp_difference/max": 0.06748957565675179, + "sampling/sampling_logp_difference/mean": 0.001276336494872036, + "step": 11160, + "step_time": 4.480735058011487, + "student/entropy": 0.04329327641365429 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.166666666666667, + "completions/max_terminated_length": 5.166666666666667, + "completions/mean_length": 1.0541666865348815, + "completions/mean_terminated_length": 1.0541666865348815, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04997519676884015, + "epoch": 0.4249420878745301, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.983070373535156, + "kd/policy_loss": -1.3311217625935872, + "kd/rkl_advantage_mean": 37.28581873575846, + "kd/rkl_advantage_p95": 50.854276871681215, + "kd/rkl_advantage_std": 21.734958377480506, + "kd/ssd_loss": 0.0, + "learning_rate": 5.750958872897125e-07, + "loss": -5.3244873046875, + "num_tokens": 141589814.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0326019962628683, + "sampling/importance_sampling_ratio/mean": 1.000212194522222, + "sampling/importance_sampling_ratio/min": 0.9500235974788666, + "sampling/sampling_logp_difference/max": 0.08107244477917751, + "sampling/sampling_logp_difference/mean": 0.0014125037462993835, + "step": 11190, + "step_time": 4.353337911128377, + "student/entropy": 0.04997519676884015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.075277813275655, + "completions/mean_terminated_length": 1.075277813275655, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054238381205747525, + "epoch": 0.4260813428018076, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.312442779541016, + "kd/policy_loss": -4.188503996531169, + "kd/rkl_advantage_mean": 61.4635247707367, + "kd/rkl_advantage_p95": 79.05435166358947, + "kd/rkl_advantage_std": 29.731412819027902, + "kd/ssd_loss": 0.0, + "learning_rate": 5.73956632362435e-07, + "loss": -16.754015096028645, + "num_tokens": 141897477.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.034173270066579, + "sampling/importance_sampling_ratio/mean": 0.999820746978124, + "sampling/importance_sampling_ratio/min": 0.9183193951845169, + "sampling/sampling_logp_difference/max": 0.12344514004265268, + "sampling/sampling_logp_difference/mean": 0.001782611333571064, + "step": 11220, + "step_time": 4.448040739163601, + "student/entropy": 0.054238381205747525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.5, + "completions/max_terminated_length": 7.5, + "completions/mean_length": 1.0794444680213928, + "completions/mean_terminated_length": 1.0794444680213928, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04554704405988256, + "epoch": 0.42722059772908516, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.99761199951172, + "kd/policy_loss": -4.304628849029541, + "kd/rkl_advantage_mean": 66.54006101290385, + "kd/rkl_advantage_p95": 84.47356358369191, + "kd/rkl_advantage_std": 32.15288753509522, + "kd/ssd_loss": 0.0, + "learning_rate": 5.728173774351573e-07, + "loss": -17.218509928385416, + "num_tokens": 142200283.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0218620856602987, + "sampling/importance_sampling_ratio/mean": 0.9997509141763051, + "sampling/importance_sampling_ratio/min": 0.9292359431584676, + "sampling/sampling_logp_difference/max": 0.09203045549802483, + "sampling/sampling_logp_difference/mean": 0.0016135405438641706, + "step": 11250, + "step_time": 4.381336424457064, + "student/entropy": 0.04554704405988256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.233333333333333, + "completions/max_terminated_length": 6.233333333333333, + "completions/mean_length": 1.0566666920979817, + "completions/mean_terminated_length": 1.0566666920979817, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04370097421730558, + "epoch": 0.4283598526563627, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.897061347961426, + "kd/policy_loss": -1.904778520266215, + "kd/rkl_advantage_mean": 50.484585507710776, + "kd/rkl_advantage_p95": 67.93998798529307, + "kd/rkl_advantage_std": 29.177558104197185, + "kd/ssd_loss": 0.0, + "learning_rate": 5.716781225078798e-07, + "loss": -7.61911366780599, + "num_tokens": 142503687.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0122064510981241, + "sampling/importance_sampling_ratio/mean": 0.9996810118357341, + "sampling/importance_sampling_ratio/min": 0.9245942374070485, + "sampling/sampling_logp_difference/max": 0.08191923612418274, + "sampling/sampling_logp_difference/mean": 0.0012023317326869194, + "step": 11280, + "step_time": 4.402948137182587, + "student/entropy": 0.04370097421730558 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.0772222558657327, + "completions/mean_terminated_length": 1.0772222558657327, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05159459998831153, + "epoch": 0.4294991075836403, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.243715286254883, + "kd/policy_loss": -2.91313449939092, + "kd/rkl_advantage_mean": 59.27784210840861, + "kd/rkl_advantage_p95": 78.83904808362325, + "kd/rkl_advantage_std": 32.860830559333166, + "kd/ssd_loss": 0.0, + "learning_rate": 5.705388675806022e-07, + "loss": -11.6525390625, + "num_tokens": 142813973.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0184686501820883, + "sampling/importance_sampling_ratio/mean": 0.9997023622194926, + "sampling/importance_sampling_ratio/min": 0.9263975004355113, + "sampling/sampling_logp_difference/max": 0.11008650118795534, + "sampling/sampling_logp_difference/mean": 0.0016727508865490867, + "step": 11310, + "step_time": 4.370177285113217, + "student/entropy": 0.05159459998831153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.0588889122009277, + "completions/mean_terminated_length": 1.0588889122009277, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.037748931891595325, + "epoch": 0.43063836251091786, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.676097869873047, + "kd/policy_loss": -1.9748494823773701, + "kd/rkl_advantage_mean": 55.24795654614766, + "kd/rkl_advantage_p95": 73.69602178732553, + "kd/rkl_advantage_std": 31.336934000253677, + "kd/ssd_loss": 0.0, + "learning_rate": 5.693996126533247e-07, + "loss": -7.899394734700521, + "num_tokens": 143117113.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0222793896993, + "sampling/importance_sampling_ratio/mean": 1.0000364104906718, + "sampling/importance_sampling_ratio/min": 0.9651066939036052, + "sampling/sampling_logp_difference/max": 0.055526341001192726, + "sampling/sampling_logp_difference/mean": 0.0007603436820015001, + "step": 11340, + "step_time": 4.304724631873736, + "student/entropy": 0.037748931891595325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.833333333333333, + "completions/max_terminated_length": 6.833333333333333, + "completions/mean_length": 1.0713889161745707, + "completions/mean_terminated_length": 1.0713889161745707, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048090742186953625, + "epoch": 0.4317776174381954, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.26636791229248, + "kd/policy_loss": -3.2468138933181763, + "kd/rkl_advantage_mean": 60.09271732966105, + "kd/rkl_advantage_p95": 78.97346949577332, + "kd/rkl_advantage_std": 31.78866969048977, + "kd/ssd_loss": 0.0, + "learning_rate": 5.682603577260472e-07, + "loss": -12.987259928385416, + "num_tokens": 143428634.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0509298324584961, + "sampling/importance_sampling_ratio/mean": 1.0001258492469787, + "sampling/importance_sampling_ratio/min": 0.9486146608988444, + "sampling/sampling_logp_difference/max": 0.08641674142951766, + "sampling/sampling_logp_difference/mean": 0.0015011152194347233, + "step": 11370, + "step_time": 4.50744074252046, + "student/entropy": 0.048090742186953625 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.433333333333334, + "completions/max_terminated_length": 5.433333333333334, + "completions/mean_length": 1.0527777989705405, + "completions/mean_terminated_length": 1.0527777989705405, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.051365650321046515, + "epoch": 0.432916872365473, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.386240005493164, + "kd/policy_loss": -0.6496706167856853, + "kd/rkl_advantage_mean": 36.97761158943176, + "kd/rkl_advantage_p95": 50.8125847975413, + "kd/rkl_advantage_std": 22.873974876602492, + "kd/ssd_loss": 0.0, + "learning_rate": 5.671211027987696e-07, + "loss": -2.5986813863118488, + "num_tokens": 143748952.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0284269769986472, + "sampling/importance_sampling_ratio/mean": 1.0001701096693674, + "sampling/importance_sampling_ratio/min": 0.9638070782025655, + "sampling/sampling_logp_difference/max": 0.06924357203145821, + "sampling/sampling_logp_difference/mean": 0.0011381190997781232, + "step": 11400, + "step_time": 4.354419611891111, + "student/entropy": 0.051365650321046515 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.866666666666666, + "completions/max_terminated_length": 7.866666666666666, + "completions/mean_length": 1.087777817249298, + "completions/mean_terminated_length": 1.087777817249298, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06163343194251259, + "epoch": 0.43405612729275056, + "eval/gt_acc_batch": 0.0025000001303851606, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.32806968688965, + "kd/policy_loss": -5.014571171998978, + "kd/rkl_advantage_mean": 73.64227692286174, + "kd/rkl_advantage_p95": 96.11033616065978, + "kd/rkl_advantage_std": 36.2755621890227, + "kd/ssd_loss": 0.0, + "learning_rate": 5.65981847871492e-07, + "loss": -20.058284505208334, + "num_tokens": 144065780.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000001303851606, + "rewards/gt_logging_reward/std": 0.025585518529017765, + "sampling/importance_sampling_ratio/max": 1.0193837483723958, + "sampling/importance_sampling_ratio/mean": 0.9998852888743083, + "sampling/importance_sampling_ratio/min": 0.9156216243902843, + "sampling/sampling_logp_difference/max": 0.11265928807357947, + "sampling/sampling_logp_difference/mean": 0.0018263449979713186, + "step": 11430, + "step_time": 4.4029921295431755, + "student/entropy": 0.06163343194251259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.666666666666667, + "completions/max_terminated_length": 7.666666666666667, + "completions/mean_length": 1.0669444719950358, + "completions/mean_terminated_length": 1.0669444719950358, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05027566875020663, + "epoch": 0.4351953822200281, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 32.90837860107422, + "kd/policy_loss": -2.83678963581721, + "kd/rkl_advantage_mean": 68.14005338350931, + "kd/rkl_advantage_p95": 90.5938871939977, + "kd/rkl_advantage_std": 38.55388252139092, + "kd/ssd_loss": 0.0, + "learning_rate": 5.648425929442144e-07, + "loss": -11.347158813476563, + "num_tokens": 144369533.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0502660830815633, + "sampling/importance_sampling_ratio/mean": 1.0001077274481456, + "sampling/importance_sampling_ratio/min": 0.9414719462394714, + "sampling/sampling_logp_difference/max": 0.07638570424169303, + "sampling/sampling_logp_difference/mean": 0.0012955533825637152, + "step": 11460, + "step_time": 4.282859873853158, + "student/entropy": 0.05027566875020663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.533333333333333, + "completions/max_terminated_length": 5.533333333333333, + "completions/mean_length": 1.0483333547910054, + "completions/mean_terminated_length": 1.0483333547910054, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03881508430155615, + "epoch": 0.43633463714730564, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 48.73698425292969, + "kd/policy_loss": -1.2100711107254027, + "kd/rkl_advantage_mean": 41.79185279210409, + "kd/rkl_advantage_p95": 56.74044295946757, + "kd/rkl_advantage_std": 25.265738673011462, + "kd/ssd_loss": 0.0, + "learning_rate": 5.637033380169369e-07, + "loss": -4.840283203125, + "num_tokens": 144682619.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0213432709376018, + "sampling/importance_sampling_ratio/mean": 1.0000312507152558, + "sampling/importance_sampling_ratio/min": 0.9632064223289489, + "sampling/sampling_logp_difference/max": 0.050279433652758596, + "sampling/sampling_logp_difference/mean": 0.0009088833365240134, + "step": 11490, + "step_time": 4.481257567531429, + "student/entropy": 0.03881508430155615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0555555820465088, + "completions/mean_terminated_length": 1.0555555820465088, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04924609437584877, + "epoch": 0.4374738920745832, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.912052154541016, + "kd/policy_loss": -1.9455425222714742, + "kd/rkl_advantage_mean": 54.18112618128459, + "kd/rkl_advantage_p95": 73.63508859475454, + "kd/rkl_advantage_std": 32.12640942335129, + "kd/ssd_loss": 0.0, + "learning_rate": 5.625640830896594e-07, + "loss": -7.7821706136067705, + "num_tokens": 144984579.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0375860651334128, + "sampling/importance_sampling_ratio/mean": 1.000180294116338, + "sampling/importance_sampling_ratio/min": 0.9582563698291778, + "sampling/sampling_logp_difference/max": 0.07530404732873043, + "sampling/sampling_logp_difference/mean": 0.0012020249171958615, + "step": 11520, + "step_time": 4.397375645507903, + "student/entropy": 0.04924609437584877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.0725000222524008, + "completions/mean_terminated_length": 1.0725000222524008, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04911743557701508, + "epoch": 0.43861314700186077, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.529566764831543, + "kd/policy_loss": -3.0544857223828634, + "kd/rkl_advantage_mean": 60.9979487101237, + "kd/rkl_advantage_p95": 79.1540152390798, + "kd/rkl_advantage_std": 31.7808699965477, + "kd/ssd_loss": 0.0, + "learning_rate": 5.614248281623818e-07, + "loss": -12.2179443359375, + "num_tokens": 145291936.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.020017679532369, + "sampling/importance_sampling_ratio/mean": 0.999701992670695, + "sampling/importance_sampling_ratio/min": 0.9296308040618897, + "sampling/sampling_logp_difference/max": 0.08415805230227609, + "sampling/sampling_logp_difference/mean": 0.0013900214699485028, + "step": 11550, + "step_time": 4.39505658429504, + "student/entropy": 0.04911743557701508 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0005555555845300357, + "completions/max_length": 39.4, + "completions/max_terminated_length": 5.7, + "completions/mean_length": 1.3333333730697632, + "completions/mean_terminated_length": 1.0494701385498046, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04944254864628116, + "epoch": 0.43975240192913834, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.808009147644043, + "kd/policy_loss": -1.2664933701356251, + "kd/rkl_advantage_mean": 48.148320023218794, + "kd/rkl_advantage_p95": 62.480532908439635, + "kd/rkl_advantage_std": 24.067156744003295, + "kd/ssd_loss": 0.0, + "learning_rate": 5.602855732351042e-07, + "loss": -5.0659734090169275, + "num_tokens": 145604064.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0527061502138773, + "sampling/importance_sampling_ratio/mean": 1.000235342979431, + "sampling/importance_sampling_ratio/min": 0.9600456317265829, + "sampling/sampling_logp_difference/max": 0.07140596597455443, + "sampling/sampling_logp_difference/mean": 0.0010531873607154314, + "step": 11580, + "step_time": 4.540124664308193, + "student/entropy": 0.04944254864628116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.833333333333333, + "completions/max_terminated_length": 7.833333333333333, + "completions/mean_length": 1.0741666952768962, + "completions/mean_terminated_length": 1.0741666952768962, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05984186812614401, + "epoch": 0.4408916568564159, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.76508140563965, + "kd/policy_loss": -2.7528430143992106, + "kd/rkl_advantage_mean": 67.28696103096009, + "kd/rkl_advantage_p95": 90.43400893211364, + "kd/rkl_advantage_std": 38.08000577290853, + "kd/ssd_loss": 0.0, + "learning_rate": 5.591463183078267e-07, + "loss": -11.011372884114584, + "num_tokens": 145906067.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0051904757817587, + "sampling/importance_sampling_ratio/mean": 0.9995664378007253, + "sampling/importance_sampling_ratio/min": 0.9198826332887013, + "sampling/sampling_logp_difference/max": 0.07929244134575128, + "sampling/sampling_logp_difference/mean": 0.0014333160875442748, + "step": 11610, + "step_time": 4.313558578192412, + "student/entropy": 0.05984186812614401 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.2, + "completions/max_terminated_length": 7.2, + "completions/mean_length": 1.0597222487131754, + "completions/mean_terminated_length": 1.0597222487131754, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04406884691367547, + "epoch": 0.44203091178369347, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.984448432922363, + "kd/policy_loss": -2.0132682363192242, + "kd/rkl_advantage_mean": 58.8227871576945, + "kd/rkl_advantage_p95": 78.87181780338287, + "kd/rkl_advantage_std": 34.06410463253657, + "kd/ssd_loss": 0.0, + "learning_rate": 5.580070633805491e-07, + "loss": -8.053070068359375, + "num_tokens": 146219834.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0191957632700601, + "sampling/importance_sampling_ratio/mean": 0.9996463894844055, + "sampling/importance_sampling_ratio/min": 0.9361909071604411, + "sampling/sampling_logp_difference/max": 0.0930317122489214, + "sampling/sampling_logp_difference/mean": 0.0013776014389198584, + "step": 11640, + "step_time": 4.30121609804919, + "student/entropy": 0.04406884691367547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.266666666666667, + "completions/max_terminated_length": 7.266666666666667, + "completions/mean_length": 1.0850000262260437, + "completions/mean_terminated_length": 1.0850000262260437, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.056204996071755886, + "epoch": 0.44317016671097104, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.872465133666992, + "kd/policy_loss": -3.801303541660309, + "kd/rkl_advantage_mean": 58.111153745651244, + "kd/rkl_advantage_p95": 73.5870222568512, + "kd/rkl_advantage_std": 27.12506257990996, + "kd/ssd_loss": 0.0, + "learning_rate": 5.568678084532715e-07, + "loss": -15.205214436848959, + "num_tokens": 146529236.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0252814094225566, + "sampling/importance_sampling_ratio/mean": 0.999856009085973, + "sampling/importance_sampling_ratio/min": 0.9326556066672007, + "sampling/sampling_logp_difference/max": 0.07547542708925903, + "sampling/sampling_logp_difference/mean": 0.001358013284819511, + "step": 11670, + "step_time": 4.315039117626536, + "student/entropy": 0.056204996071755886 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.466666666666667, + "completions/max_terminated_length": 7.466666666666667, + "completions/mean_length": 1.073055585225423, + "completions/mean_terminated_length": 1.073055585225423, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04646641332656145, + "epoch": 0.4443094216382486, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.204472541809082, + "kd/policy_loss": -3.4098310987154643, + "kd/rkl_advantage_mean": 65.09974377950033, + "kd/rkl_advantage_p95": 85.05006685256959, + "kd/rkl_advantage_std": 34.333190030852954, + "kd/ssd_loss": 0.0, + "learning_rate": 5.55728553525994e-07, + "loss": -13.63932596842448, + "num_tokens": 146842387.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0258016546567281, + "sampling/importance_sampling_ratio/mean": 0.999952616294225, + "sampling/importance_sampling_ratio/min": 0.957244332631429, + "sampling/sampling_logp_difference/max": 0.06862794890378912, + "sampling/sampling_logp_difference/mean": 0.0011243236891459673, + "step": 11700, + "step_time": 4.395958664287658, + "student/entropy": 0.04646641332656145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.433333333333334, + "completions/max_terminated_length": 5.433333333333334, + "completions/mean_length": 1.0450000206629435, + "completions/mean_terminated_length": 1.0450000206629435, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0318005400399367, + "epoch": 0.44544867656552617, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.205329895019531, + "kd/policy_loss": -0.8154112021128337, + "kd/rkl_advantage_mean": 40.48317550023397, + "kd/rkl_advantage_p95": 56.508413537343344, + "kd/rkl_advantage_std": 25.986789824565253, + "kd/ssd_loss": 0.0, + "learning_rate": 5.545892985987164e-07, + "loss": -3.261644236246745, + "num_tokens": 147155637.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.03867666721344, + "sampling/importance_sampling_ratio/mean": 1.000117427110672, + "sampling/importance_sampling_ratio/min": 0.9657951970895131, + "sampling/sampling_logp_difference/max": 0.06877054933769007, + "sampling/sampling_logp_difference/mean": 0.000993752298381878, + "step": 11730, + "step_time": 4.434804563918927, + "student/entropy": 0.0318005400399367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.366666666666666, + "completions/max_terminated_length": 6.366666666666666, + "completions/mean_length": 1.0747222503026326, + "completions/mean_terminated_length": 1.0747222503026326, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04802000230799119, + "epoch": 0.4465879314928037, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.057693481445312, + "kd/policy_loss": -3.031216561794281, + "kd/rkl_advantage_mean": 52.14178721110026, + "kd/rkl_advantage_p95": 68.0062349875768, + "kd/rkl_advantage_std": 26.69708133637905, + "kd/ssd_loss": 0.0, + "learning_rate": 5.534500436714389e-07, + "loss": -12.124867757161459, + "num_tokens": 147463610.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0319310744603476, + "sampling/importance_sampling_ratio/mean": 1.0000098447004955, + "sampling/importance_sampling_ratio/min": 0.9536249220371247, + "sampling/sampling_logp_difference/max": 0.06417330106099446, + "sampling/sampling_logp_difference/mean": 0.0011972082972836991, + "step": 11760, + "step_time": 4.481243357093384, + "student/entropy": 0.04802000230799119 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.733333333333333, + "completions/max_terminated_length": 7.733333333333333, + "completions/mean_length": 1.0816667000452678, + "completions/mean_terminated_length": 1.0816667000452678, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05498358129213254, + "epoch": 0.44772718642008125, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 35.11229705810547, + "kd/policy_loss": -3.990749883651733, + "kd/rkl_advantage_mean": 69.7995202700297, + "kd/rkl_advantage_p95": 90.23993740081787, + "kd/rkl_advantage_std": 35.58574637671312, + "kd/ssd_loss": 0.0, + "learning_rate": 5.523107887441613e-07, + "loss": -15.962999471028645, + "num_tokens": 147780952.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0228901863098145, + "sampling/importance_sampling_ratio/mean": 0.9995510637760162, + "sampling/importance_sampling_ratio/min": 0.9191923836867014, + "sampling/sampling_logp_difference/max": 0.09243130329996348, + "sampling/sampling_logp_difference/mean": 0.0015913472811613853, + "step": 11790, + "step_time": 4.454244124571172, + "student/entropy": 0.05498358129213254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.833333333333333, + "completions/max_terminated_length": 6.833333333333333, + "completions/mean_length": 1.0513889074325562, + "completions/mean_terminated_length": 1.0513889074325562, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04336817987884085, + "epoch": 0.4488664413473588, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.824317932128906, + "kd/policy_loss": -0.9272020022074382, + "kd/rkl_advantage_mean": 49.89709688822428, + "kd/rkl_advantage_p95": 67.62741022109985, + "kd/rkl_advantage_std": 30.317295532425245, + "kd/ssd_loss": 0.0, + "learning_rate": 5.511715338168837e-07, + "loss": -3.708807373046875, + "num_tokens": 148087969.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.029088318347931, + "sampling/importance_sampling_ratio/mean": 0.9996407230695089, + "sampling/importance_sampling_ratio/min": 0.9197608321905136, + "sampling/sampling_logp_difference/max": 0.09190168903830151, + "sampling/sampling_logp_difference/mean": 0.0015204577435118456, + "step": 11820, + "step_time": 4.370895058630655, + "student/entropy": 0.04336817987884085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.633333333333334, + "completions/max_terminated_length": 5.633333333333334, + "completions/mean_length": 1.0480555772781373, + "completions/mean_terminated_length": 1.0480555772781373, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05310496448849638, + "epoch": 0.4500056962746364, + "eval/gt_acc_batch": 0.0008333334078391393, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.42755126953125, + "kd/policy_loss": -0.6529990096886953, + "kd/rkl_advantage_mean": 42.38942914009094, + "kd/rkl_advantage_p95": 61.6935622771581, + "kd/rkl_advantage_std": 29.133059361577033, + "kd/ssd_loss": 0.0, + "learning_rate": 5.500322788896061e-07, + "loss": -2.6119959513346354, + "num_tokens": 148398110.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333457509677, + "rewards/gt_logging_reward/std": 0.0052259857455889385, + "sampling/importance_sampling_ratio/max": 1.0265888532002767, + "sampling/importance_sampling_ratio/mean": 1.0000197827816009, + "sampling/importance_sampling_ratio/min": 0.9707099974155426, + "sampling/sampling_logp_difference/max": 0.06003935287396113, + "sampling/sampling_logp_difference/mean": 0.0010824763351896156, + "step": 11850, + "step_time": 4.303925103150929, + "student/entropy": 0.05310496448849638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.0519444624582925, + "completions/mean_terminated_length": 1.0519444624582925, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05675328066572547, + "epoch": 0.45114495120191395, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 102.36341857910156, + "kd/policy_loss": -1.227269168694814, + "kd/rkl_advantage_mean": 45.407608604431154, + "kd/rkl_advantage_p95": 61.98249460061391, + "kd/rkl_advantage_std": 27.611625778178375, + "kd/ssd_loss": 0.0, + "learning_rate": 5.488930239623286e-07, + "loss": -4.909075927734375, + "num_tokens": 148708857.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0848882039388021, + "sampling/importance_sampling_ratio/mean": 1.0005745967229207, + "sampling/importance_sampling_ratio/min": 0.9619377652804056, + "sampling/sampling_logp_difference/max": 0.0782346207027634, + "sampling/sampling_logp_difference/mean": 0.001386181966518052, + "step": 11880, + "step_time": 4.483617608707088, + "student/entropy": 0.05675328066572547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.4, + "completions/max_terminated_length": 7.4, + "completions/mean_length": 1.0888889153798422, + "completions/mean_terminated_length": 1.0888889153798422, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07168546644970775, + "epoch": 0.4522842061291915, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 75.8016357421875, + "kd/policy_loss": -2.762350622812907, + "kd/rkl_advantage_mean": 56.737461996078494, + "kd/rkl_advantage_p95": 73.63984711170197, + "kd/rkl_advantage_std": 29.290853317578634, + "kd/ssd_loss": 0.0, + "learning_rate": 5.477537690350512e-07, + "loss": -11.049403889973958, + "num_tokens": 149010161.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0599310755729676, + "sampling/importance_sampling_ratio/mean": 1.000016717116038, + "sampling/importance_sampling_ratio/min": 0.9407779892285665, + "sampling/sampling_logp_difference/max": 0.09746883294234673, + "sampling/sampling_logp_difference/mean": 0.001858974409212048, + "step": 11910, + "step_time": 4.387714311092471, + "student/entropy": 0.07168546644970775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.1, + "completions/max_terminated_length": 7.1, + "completions/mean_length": 1.0686111370722453, + "completions/mean_terminated_length": 1.0686111370722453, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06051940399532517, + "epoch": 0.4534234610564691, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.720794677734375, + "kd/policy_loss": -2.5508042375246682, + "kd/rkl_advantage_mean": 62.33538484573364, + "kd/rkl_advantage_p95": 84.51406393051147, + "kd/rkl_advantage_std": 36.09601806253195, + "kd/ssd_loss": 0.0, + "learning_rate": 5.466145141077735e-07, + "loss": -10.203218587239583, + "num_tokens": 149317160.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0445067882537842, + "sampling/importance_sampling_ratio/mean": 1.0000900447368621, + "sampling/importance_sampling_ratio/min": 0.9658092856407166, + "sampling/sampling_logp_difference/max": 0.0723790698684752, + "sampling/sampling_logp_difference/mean": 0.0012795442395145073, + "step": 11940, + "step_time": 4.382093028475841, + "student/entropy": 0.06051940399532517 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0844444712003072, + "completions/mean_terminated_length": 1.0844444712003072, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06952894149969022, + "epoch": 0.45456271598374665, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 56.45022201538086, + "kd/policy_loss": -3.2257046937942504, + "kd/rkl_advantage_mean": 44.70605607032776, + "kd/rkl_advantage_p95": 56.87268641789754, + "kd/rkl_advantage_std": 20.178324483831723, + "kd/ssd_loss": 0.0, + "learning_rate": 5.45475259180496e-07, + "loss": -12.902817789713541, + "num_tokens": 149623584.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0460597117741903, + "sampling/importance_sampling_ratio/mean": 1.000152732928594, + "sampling/importance_sampling_ratio/min": 0.9664196570714315, + "sampling/sampling_logp_difference/max": 0.061068541013325255, + "sampling/sampling_logp_difference/mean": 0.001816092574154027, + "step": 11970, + "step_time": 4.360609302840506, + "student/entropy": 0.06952894149969022 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.233333333333333, + "completions/max_terminated_length": 8.233333333333333, + "completions/mean_length": 1.1150000413258871, + "completions/mean_terminated_length": 1.1150000413258871, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07108267511551579, + "epoch": 0.4557019709110242, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 142.2810821533203, + "kd/policy_loss": -6.0142657399177555, + "kd/rkl_advantage_mean": 68.81414844195048, + "kd/rkl_advantage_p95": 84.94450029532115, + "kd/rkl_advantage_std": 27.95167461435, + "kd/ssd_loss": 0.0, + "learning_rate": 5.443360042532184e-07, + "loss": -24.057063802083334, + "num_tokens": 149929654.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0760665814081827, + "sampling/importance_sampling_ratio/mean": 0.9999898493289947, + "sampling/importance_sampling_ratio/min": 0.9225591540336608, + "sampling/sampling_logp_difference/max": 0.12508328657907744, + "sampling/sampling_logp_difference/mean": 0.0024773817334789784, + "step": 12000, + "step_time": 4.488526214372056, + "student/entropy": 0.07108267511551579 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.366666666666667, + "completions/max_terminated_length": 8.366666666666667, + "completions/mean_length": 1.1013889233271281, + "completions/mean_terminated_length": 1.1013889233271281, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.060439442346493406, + "epoch": 0.4568412258383017, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.475035667419434, + "kd/policy_loss": -4.484951303402583, + "kd/rkl_advantage_mean": 70.53711113929748, + "kd/rkl_advantage_p95": 90.11047903696696, + "kd/rkl_advantage_std": 33.962203763922055, + "kd/ssd_loss": 0.0, + "learning_rate": 5.431967493259408e-07, + "loss": -17.93980712890625, + "num_tokens": 150230539.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0260621746381124, + "sampling/importance_sampling_ratio/mean": 0.9995723942915599, + "sampling/importance_sampling_ratio/min": 0.917457906405131, + "sampling/sampling_logp_difference/max": 0.11420298611434797, + "sampling/sampling_logp_difference/mean": 0.0019092475393942246, + "step": 12030, + "step_time": 4.339861938326309, + "student/entropy": 0.060439442346493406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 22.533333333333335, + "completions/max_terminated_length": 5.5, + "completions/mean_length": 1.18194446961085, + "completions/mean_terminated_length": 1.0400000135103862, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.053486026668300234, + "epoch": 0.4579804807655793, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 34.653289794921875, + "kd/policy_loss": 1.0815146565437317, + "kd/rkl_advantage_mean": 31.501799949010213, + "kd/rkl_advantage_p95": 44.824370034535725, + "kd/rkl_advantage_std": 21.746348209182422, + "kd/ssd_loss": 0.0, + "learning_rate": 5.420574943986632e-07, + "loss": 4.326058959960937, + "num_tokens": 150536954.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0601528763771058, + "sampling/importance_sampling_ratio/mean": 1.0005824228127798, + "sampling/importance_sampling_ratio/min": 0.9942478974660237, + "sampling/sampling_logp_difference/max": 0.06238632166447739, + "sampling/sampling_logp_difference/mean": 0.0010172936842233564, + "step": 12060, + "step_time": 4.4161894454310335, + "student/entropy": 0.053486026668300234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.733333333333333, + "completions/max_terminated_length": 8.733333333333333, + "completions/mean_length": 1.1055555979410807, + "completions/mean_terminated_length": 1.1055555979410807, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07043387883653243, + "epoch": 0.45911973569285686, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 51.702850341796875, + "kd/policy_loss": -6.108171383539836, + "kd/rkl_advantage_mean": 85.1475962003072, + "kd/rkl_advantage_p95": 107.43587476412455, + "kd/rkl_advantage_std": 39.82702850302061, + "kd/ssd_loss": 0.0, + "learning_rate": 5.409182394713857e-07, + "loss": -24.432686360677085, + "num_tokens": 150840990.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0410391290982564, + "sampling/importance_sampling_ratio/mean": 0.9998147269090016, + "sampling/importance_sampling_ratio/min": 0.9110970298449198, + "sampling/sampling_logp_difference/max": 0.0891763656710585, + "sampling/sampling_logp_difference/mean": 0.0018504360758621865, + "step": 12090, + "step_time": 4.354118770166921, + "student/entropy": 0.07043387883653243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.666666666666667, + "completions/max_terminated_length": 6.666666666666667, + "completions/mean_length": 1.0605555772781372, + "completions/mean_terminated_length": 1.0605555772781372, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04486687825992704, + "epoch": 0.4602589906201344, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.822619438171387, + "kd/policy_loss": -1.2763205250104268, + "kd/rkl_advantage_mean": 53.258034737904865, + "kd/rkl_advantage_p95": 73.18954383532206, + "kd/rkl_advantage_std": 33.0258485386769, + "kd/ssd_loss": 0.0, + "learning_rate": 5.397789845441081e-07, + "loss": -5.105282592773437, + "num_tokens": 151139024.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.022306215763092, + "sampling/importance_sampling_ratio/mean": 0.9997896512349447, + "sampling/importance_sampling_ratio/min": 0.9510839243729909, + "sampling/sampling_logp_difference/max": 0.06029369751922786, + "sampling/sampling_logp_difference/mean": 0.0009003585820513156, + "step": 12120, + "step_time": 4.568763402302284, + "student/entropy": 0.04486687825992704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.05444446404775, + "completions/mean_terminated_length": 1.05444446404775, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0486093921897312, + "epoch": 0.461398245547412, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.555797576904297, + "kd/policy_loss": -1.1853406270345053, + "kd/rkl_advantage_mean": 45.693617534637454, + "kd/rkl_advantage_p95": 62.11831531524658, + "kd/rkl_advantage_std": 27.511588749289512, + "kd/ssd_loss": 0.0, + "learning_rate": 5.386397296168306e-07, + "loss": -4.741361999511719, + "num_tokens": 151440828.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0213158051172893, + "sampling/importance_sampling_ratio/mean": 0.9999685247739156, + "sampling/importance_sampling_ratio/min": 0.9717375377813975, + "sampling/sampling_logp_difference/max": 0.044946848042309286, + "sampling/sampling_logp_difference/mean": 0.0008553359997070705, + "step": 12150, + "step_time": 4.70310128138711, + "student/entropy": 0.0486093921897312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.033333333333333, + "completions/max_terminated_length": 7.033333333333333, + "completions/mean_length": 1.076388919353485, + "completions/mean_terminated_length": 1.076388919353485, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06322260800128182, + "epoch": 0.46253750047468956, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.630434036254883, + "kd/policy_loss": -2.815470878283183, + "kd/rkl_advantage_mean": 56.203433958689374, + "kd/rkl_advantage_p95": 73.23380906581879, + "kd/rkl_advantage_std": 29.39581671456496, + "kd/ssd_loss": 0.0, + "learning_rate": 5.375004746895531e-07, + "loss": -11.261881510416666, + "num_tokens": 151747831.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.1022727251052857, + "sampling/importance_sampling_ratio/mean": 1.0002849042415618, + "sampling/importance_sampling_ratio/min": 0.9416389962037405, + "sampling/sampling_logp_difference/max": 0.12190096022871634, + "sampling/sampling_logp_difference/mean": 0.0020285154818945254, + "step": 12180, + "step_time": 4.534224947289719, + "student/entropy": 0.06322260800128182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 22.566666666666666, + "completions/max_terminated_length": 5.533333333333333, + "completions/mean_length": 1.1866666992505392, + "completions/mean_terminated_length": 1.0447222431500753, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054340109198043746, + "epoch": 0.4636767554019671, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.824872970581055, + "kd/policy_loss": -0.3583298047383626, + "kd/rkl_advantage_mean": 44.628721968332925, + "kd/rkl_advantage_p95": 61.983014265696205, + "kd/rkl_advantage_std": 27.80812741567691, + "kd/ssd_loss": 0.0, + "learning_rate": 5.363612197622754e-07, + "loss": -1.4333198547363282, + "num_tokens": 152059359.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0434519012769063, + "sampling/importance_sampling_ratio/mean": 1.0000710646311441, + "sampling/importance_sampling_ratio/min": 0.9597547670205434, + "sampling/sampling_logp_difference/max": 0.08065934148617089, + "sampling/sampling_logp_difference/mean": 0.0011759459594031796, + "step": 12210, + "step_time": 4.564156152117842, + "student/entropy": 0.054340109198043746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.833333333333333, + "completions/max_terminated_length": 7.833333333333333, + "completions/mean_length": 1.0813889265060426, + "completions/mean_terminated_length": 1.0813889265060426, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05818590925385555, + "epoch": 0.4648160103292447, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.737817764282227, + "kd/policy_loss": -4.3177132427692415, + "kd/rkl_advantage_mean": 72.62354628245036, + "kd/rkl_advantage_p95": 95.69041108290354, + "kd/rkl_advantage_std": 37.95568589518468, + "kd/ssd_loss": 0.0, + "learning_rate": 5.352219648349979e-07, + "loss": -17.270853678385418, + "num_tokens": 152368364.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.059200644493103, + "sampling/importance_sampling_ratio/mean": 1.0000500778357189, + "sampling/importance_sampling_ratio/min": 0.9377648095289867, + "sampling/sampling_logp_difference/max": 0.084792913030833, + "sampling/sampling_logp_difference/mean": 0.0014539069447588797, + "step": 12240, + "step_time": 4.494063816173002, + "student/entropy": 0.05818590925385555 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 22.566666666666666, + "completions/max_terminated_length": 5.933333333333334, + "completions/mean_length": 1.1913889169692993, + "completions/mean_terminated_length": 1.0494724790255228, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.042013262988378604, + "epoch": 0.46595526525652226, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.065299987792969, + "kd/policy_loss": -0.8000026802221935, + "kd/rkl_advantage_mean": 45.56010057131449, + "kd/rkl_advantage_p95": 62.38656537532806, + "kd/rkl_advantage_std": 27.30769639313221, + "kd/ssd_loss": 0.0, + "learning_rate": 5.340827099077203e-07, + "loss": -3.2000101725260417, + "num_tokens": 152679637.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0760546247164409, + "sampling/importance_sampling_ratio/mean": 1.0003781954447428, + "sampling/importance_sampling_ratio/min": 0.9606742690006892, + "sampling/sampling_logp_difference/max": 0.08920740114214519, + "sampling/sampling_logp_difference/mean": 0.001217976090750502, + "step": 12270, + "step_time": 4.719796882515463, + "student/entropy": 0.042013262988378604 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.0530555764834086, + "completions/mean_terminated_length": 1.0530555764834086, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05238965703174472, + "epoch": 0.4670945201837998, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.381067276000977, + "kd/policy_loss": -0.5744364341100057, + "kd/rkl_advantage_mean": 44.711557944615684, + "kd/rkl_advantage_p95": 62.26100041071574, + "kd/rkl_advantage_std": 28.975821004311243, + "kd/ssd_loss": 0.0, + "learning_rate": 5.329434549804428e-07, + "loss": -2.2977472941080728, + "num_tokens": 152999044.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0070522546768188, + "sampling/importance_sampling_ratio/mean": 0.9997647066911062, + "sampling/importance_sampling_ratio/min": 0.9573486864566803, + "sampling/sampling_logp_difference/max": 0.06475061339636644, + "sampling/sampling_logp_difference/mean": 0.0011575575753037506, + "step": 12300, + "step_time": 4.651913744269405, + "student/entropy": 0.05238965703174472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.1, + "completions/max_terminated_length": 8.1, + "completions/mean_length": 1.090000033378601, + "completions/mean_terminated_length": 1.090000033378601, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0710121943304936, + "epoch": 0.46823377511107733, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.40802764892578, + "kd/policy_loss": -5.107727074623108, + "kd/rkl_advantage_mean": 72.16879695256551, + "kd/rkl_advantage_p95": 90.85547150770823, + "kd/rkl_advantage_std": 33.33307955861092, + "kd/ssd_loss": 0.0, + "learning_rate": 5.318042000531652e-07, + "loss": -20.430912272135416, + "num_tokens": 153305448.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0375855048497518, + "sampling/importance_sampling_ratio/mean": 1.0000362773736318, + "sampling/importance_sampling_ratio/min": 0.932738188902537, + "sampling/sampling_logp_difference/max": 0.08315633448461691, + "sampling/sampling_logp_difference/mean": 0.0016765917573745052, + "step": 12330, + "step_time": 4.534111306703805, + "student/entropy": 0.0710121943304936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.566666666666666, + "completions/max_terminated_length": 7.566666666666666, + "completions/mean_length": 1.076388919353485, + "completions/mean_terminated_length": 1.076388919353485, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05707078942408164, + "epoch": 0.4693730300383549, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.24410057067871, + "kd/policy_loss": -4.3454061627388, + "kd/rkl_advantage_mean": 66.34219862620036, + "kd/rkl_advantage_p95": 84.74479388395945, + "kd/rkl_advantage_std": 31.758931519587836, + "kd/ssd_loss": 0.0, + "learning_rate": 5.306649451258877e-07, + "loss": -17.381624348958333, + "num_tokens": 153613379.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0451724926630657, + "sampling/importance_sampling_ratio/mean": 0.9999876836935679, + "sampling/importance_sampling_ratio/min": 0.9422393520673116, + "sampling/sampling_logp_difference/max": 0.07991555884170036, + "sampling/sampling_logp_difference/mean": 0.0013963366664635638, + "step": 12360, + "step_time": 4.492932540368444, + "student/entropy": 0.05707078942408164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.533333333333333, + "completions/max_terminated_length": 5.533333333333333, + "completions/mean_length": 1.0469444632530212, + "completions/mean_terminated_length": 1.0469444632530212, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03855144027620554, + "epoch": 0.47051228496563247, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.876354217529297, + "kd/policy_loss": 0.18289347489674887, + "kd/rkl_advantage_mean": 38.905166180928546, + "kd/rkl_advantage_p95": 56.59665438334147, + "kd/rkl_advantage_std": 28.125340259075166, + "kd/ssd_loss": 0.0, + "learning_rate": 5.2952569019861e-07, + "loss": 0.731572977701823, + "num_tokens": 153925996.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.020102314154307, + "sampling/importance_sampling_ratio/mean": 0.9999181389808655, + "sampling/importance_sampling_ratio/min": 0.959257032473882, + "sampling/sampling_logp_difference/max": 0.05495194351921479, + "sampling/sampling_logp_difference/mean": 0.0008466461862553842, + "step": 12390, + "step_time": 4.519834479165729, + "student/entropy": 0.03855144027620554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.0850000262260437, + "completions/mean_terminated_length": 1.0850000262260437, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0442773524671793, + "epoch": 0.47165153989291003, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.500953674316406, + "kd/policy_loss": -4.569871032238007, + "kd/rkl_advantage_mean": 62.79156959851583, + "kd/rkl_advantage_p95": 79.31382265090943, + "kd/rkl_advantage_std": 28.178609525163967, + "kd/ssd_loss": 0.0, + "learning_rate": 5.283864352713325e-07, + "loss": -18.279484049479166, + "num_tokens": 154224766.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0070044596989949, + "sampling/importance_sampling_ratio/mean": 0.9995318651199341, + "sampling/importance_sampling_ratio/min": 0.9435408711433411, + "sampling/sampling_logp_difference/max": 0.06046391601363818, + "sampling/sampling_logp_difference/mean": 0.0010949551729330172, + "step": 12420, + "step_time": 4.497684118760905, + "student/entropy": 0.0442773524671793 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.0644444704055787, + "completions/mean_terminated_length": 1.0644444704055787, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.055090676372249923, + "epoch": 0.4727907948201876, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.738229751586914, + "kd/policy_loss": -2.2269298672676086, + "kd/rkl_advantage_mean": 54.214321819941205, + "kd/rkl_advantage_p95": 73.43373707930247, + "kd/rkl_advantage_std": 31.048256039619446, + "kd/ssd_loss": 0.0, + "learning_rate": 5.272471803440549e-07, + "loss": -8.907720947265625, + "num_tokens": 154523750.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0682004610697429, + "sampling/importance_sampling_ratio/mean": 1.0001738965511322, + "sampling/importance_sampling_ratio/min": 0.9440871924161911, + "sampling/sampling_logp_difference/max": 0.11092483007038632, + "sampling/sampling_logp_difference/mean": 0.0015606865073398998, + "step": 12450, + "step_time": 4.463787193379054, + "student/entropy": 0.055090676372249923 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.0777778069178263, + "completions/mean_terminated_length": 1.0777778069178263, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.060638577522089086, + "epoch": 0.47393004974746517, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 26.584558486938477, + "kd/policy_loss": -2.075624712308248, + "kd/rkl_advantage_mean": 50.94748288790385, + "kd/rkl_advantage_p95": 67.58458282947541, + "kd/rkl_advantage_std": 28.1292080839475, + "kd/ssd_loss": 0.0, + "learning_rate": 5.261079254167774e-07, + "loss": -8.302501424153645, + "num_tokens": 154827726.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0325528224309286, + "sampling/importance_sampling_ratio/mean": 0.9999174316724141, + "sampling/importance_sampling_ratio/min": 0.9441904882589977, + "sampling/sampling_logp_difference/max": 0.07085482468828559, + "sampling/sampling_logp_difference/mean": 0.0014834038127446547, + "step": 12480, + "step_time": 4.519678509805817, + "student/entropy": 0.060638577522089086 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 26.0, + "completions/max_terminated_length": 9.333333333333334, + "completions/mean_length": 1.241111147403717, + "completions/mean_terminated_length": 1.0992157220840455, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052606496835748355, + "epoch": 0.47506930467474273, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 8.739904403686523, + "kd/policy_loss": -5.581839718421301, + "kd/rkl_advantage_mean": 89.37354747454326, + "kd/rkl_advantage_p95": 113.33293160597484, + "kd/rkl_advantage_std": 42.201203137636185, + "kd/ssd_loss": 0.0, + "learning_rate": 5.249686704894999e-07, + "loss": -22.327362060546875, + "num_tokens": 155128658.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.093368943532308, + "sampling/importance_sampling_ratio/mean": 1.0002969165643056, + "sampling/importance_sampling_ratio/min": 0.9258387386798859, + "sampling/sampling_logp_difference/max": 0.12431840961799026, + "sampling/sampling_logp_difference/mean": 0.0017408357952566197, + "step": 12510, + "step_time": 4.70016779783958, + "student/entropy": 0.052606496835748355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.733333333333333, + "completions/max_terminated_length": 5.733333333333333, + "completions/mean_length": 1.0461111307144164, + "completions/mean_terminated_length": 1.0461111307144164, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.035220707192396124, + "epoch": 0.4762085596020203, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.683236122131348, + "kd/policy_loss": -0.31565155982971194, + "kd/rkl_advantage_mean": 39.89416782061259, + "kd/rkl_advantage_p95": 55.87850113709768, + "kd/rkl_advantage_std": 26.509642830491067, + "kd/ssd_loss": 0.0, + "learning_rate": 5.238294155622223e-07, + "loss": -1.2626065572102865, + "num_tokens": 155438208.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.028142786026001, + "sampling/importance_sampling_ratio/mean": 1.000002682209015, + "sampling/importance_sampling_ratio/min": 0.9581560770670573, + "sampling/sampling_logp_difference/max": 0.06265849568881095, + "sampling/sampling_logp_difference/mean": 0.0008946939885693913, + "step": 12540, + "step_time": 4.691517816503377, + "student/entropy": 0.035220707192396124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.433333333333334, + "completions/max_terminated_length": 7.433333333333334, + "completions/mean_length": 1.0813889185587564, + "completions/mean_terminated_length": 1.0813889185587564, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04845595654721061, + "epoch": 0.47734781452929786, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.842451095581055, + "kd/policy_loss": -3.338875341415405, + "kd/rkl_advantage_mean": 61.35613549550374, + "kd/rkl_advantage_p95": 79.21014640331268, + "kd/rkl_advantage_std": 30.93465841114521, + "kd/ssd_loss": 0.0, + "learning_rate": 5.226901606349448e-07, + "loss": -13.355499267578125, + "num_tokens": 155740789.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0302431066830953, + "sampling/importance_sampling_ratio/mean": 0.9996423165003459, + "sampling/importance_sampling_ratio/min": 0.9350683132807414, + "sampling/sampling_logp_difference/max": 0.09334655751784643, + "sampling/sampling_logp_difference/mean": 0.0013818571387673728, + "step": 12570, + "step_time": 4.59808313647906, + "student/entropy": 0.04845595654721061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.666666666666667, + "completions/max_terminated_length": 6.666666666666667, + "completions/mean_length": 1.0552778045336406, + "completions/mean_terminated_length": 1.0552778045336406, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04409353795150916, + "epoch": 0.4784870694565754, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.22298812866211, + "kd/policy_loss": -1.9174872597058614, + "kd/rkl_advantage_mean": 56.757350556055705, + "kd/rkl_advantage_p95": 78.47407540480296, + "kd/rkl_advantage_std": 34.72844900389512, + "kd/ssd_loss": 0.0, + "learning_rate": 5.215509057076671e-07, + "loss": -7.669948323567708, + "num_tokens": 156042836.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0142884691556295, + "sampling/importance_sampling_ratio/mean": 0.999933656056722, + "sampling/importance_sampling_ratio/min": 0.961582754055659, + "sampling/sampling_logp_difference/max": 0.04615859937233229, + "sampling/sampling_logp_difference/mean": 0.0007483174980734475, + "step": 12600, + "step_time": 4.48074091882833, + "student/entropy": 0.04409353795150916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.033333333333333, + "completions/max_terminated_length": 7.033333333333333, + "completions/mean_length": 1.0708333571751913, + "completions/mean_terminated_length": 1.0708333571751913, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04645531813924511, + "epoch": 0.47962632438385294, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 54.655738830566406, + "kd/policy_loss": -1.9541595617930094, + "kd/rkl_advantage_mean": 51.14778159459432, + "kd/rkl_advantage_p95": 67.65981026490529, + "kd/rkl_advantage_std": 28.16968336701393, + "kd/ssd_loss": 0.0, + "learning_rate": 5.204116507803896e-07, + "loss": -7.816640218098958, + "num_tokens": 156356571.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0236354231834413, + "sampling/importance_sampling_ratio/mean": 0.9998884061972301, + "sampling/importance_sampling_ratio/min": 0.9639840761820475, + "sampling/sampling_logp_difference/max": 0.05276274874340743, + "sampling/sampling_logp_difference/mean": 0.000944638312406217, + "step": 12630, + "step_time": 4.606440449727233, + "student/entropy": 0.04645531813924511 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.966666666666667, + "completions/max_terminated_length": 6.966666666666667, + "completions/mean_length": 1.0813889106114705, + "completions/mean_terminated_length": 1.0813889106114705, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.056587407179176805, + "epoch": 0.4807655793111305, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.732810974121094, + "kd/policy_loss": -3.096601498126984, + "kd/rkl_advantage_mean": 55.7760892868042, + "kd/rkl_advantage_p95": 73.2640214363734, + "kd/rkl_advantage_std": 28.665936383605004, + "kd/ssd_loss": 0.0, + "learning_rate": 5.19272395853112e-07, + "loss": -12.386405436197917, + "num_tokens": 156653840.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0346406777699788, + "sampling/importance_sampling_ratio/mean": 0.9996835788091024, + "sampling/importance_sampling_ratio/min": 0.9247465332349142, + "sampling/sampling_logp_difference/max": 0.08756525736146917, + "sampling/sampling_logp_difference/mean": 0.0015689759782011, + "step": 12660, + "step_time": 4.510720085915334, + "student/entropy": 0.056587407179176805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.433333333333334, + "completions/max_terminated_length": 7.433333333333334, + "completions/mean_length": 1.079722249507904, + "completions/mean_terminated_length": 1.079722249507904, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05163932538901766, + "epoch": 0.4819048342384081, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.964080810546875, + "kd/policy_loss": -3.6407586971918744, + "kd/rkl_advantage_mean": 62.243442090352374, + "kd/rkl_advantage_p95": 79.37273694674174, + "kd/rkl_advantage_std": 30.52121035506328, + "kd/ssd_loss": 0.0, + "learning_rate": 5.181331409258344e-07, + "loss": -14.563035074869791, + "num_tokens": 156960567.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.037150228023529, + "sampling/importance_sampling_ratio/mean": 0.999729456504186, + "sampling/importance_sampling_ratio/min": 0.9385257760683695, + "sampling/sampling_logp_difference/max": 0.0707345050914834, + "sampling/sampling_logp_difference/mean": 0.0014569775560327495, + "step": 12690, + "step_time": 4.633895054319874, + "student/entropy": 0.05163932538901766 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 23.0, + "completions/max_terminated_length": 6.3, + "completions/mean_length": 1.1991667111714681, + "completions/mean_terminated_length": 1.0572455962498983, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04691685705135266, + "epoch": 0.48304408916568564, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.078972816467285, + "kd/policy_loss": -1.455280488729477, + "kd/rkl_advantage_mean": 53.17649119695027, + "kd/rkl_advantage_p95": 73.2929831345876, + "kd/rkl_advantage_std": 31.49883407255014, + "kd/ssd_loss": 0.0, + "learning_rate": 5.169938859985569e-07, + "loss": -5.821123250325521, + "num_tokens": 157269972.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.034285851319631, + "sampling/importance_sampling_ratio/mean": 1.000057621796926, + "sampling/importance_sampling_ratio/min": 0.9663816293080648, + "sampling/sampling_logp_difference/max": 0.0628047940786928, + "sampling/sampling_logp_difference/mean": 0.0010002510340806718, + "step": 12720, + "step_time": 4.688834419082074, + "student/entropy": 0.04691685705135266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.2, + "completions/max_terminated_length": 7.2, + "completions/mean_length": 1.076388915379842, + "completions/mean_terminated_length": 1.076388915379842, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054070755404730636, + "epoch": 0.4841833440929632, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.763266563415527, + "kd/policy_loss": -3.149571434656779, + "kd/rkl_advantage_mean": 60.98614732424418, + "kd/rkl_advantage_p95": 78.9603903055191, + "kd/rkl_advantage_std": 31.830497818688553, + "kd/ssd_loss": 0.0, + "learning_rate": 5.158546310712794e-07, + "loss": -12.59828592936198, + "num_tokens": 157580551.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.045058250427246, + "sampling/importance_sampling_ratio/mean": 0.9995936493078867, + "sampling/importance_sampling_ratio/min": 0.9394316732883453, + "sampling/sampling_logp_difference/max": 0.09657004949015875, + "sampling/sampling_logp_difference/mean": 0.0017757970112143085, + "step": 12750, + "step_time": 4.577007757515336, + "student/entropy": 0.054070755404730636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.133333333333334, + "completions/max_terminated_length": 5.133333333333334, + "completions/mean_length": 1.061388913790385, + "completions/mean_terminated_length": 1.061388913790385, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05389593628545602, + "epoch": 0.4853225990202408, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 84.7264633178711, + "kd/policy_loss": -0.5527859250704448, + "kd/rkl_advantage_mean": 35.561015129089355, + "kd/rkl_advantage_p95": 50.669701178868614, + "kd/rkl_advantage_std": 23.700052812695503, + "kd/ssd_loss": 0.0, + "learning_rate": 5.147153761440018e-07, + "loss": -2.2111432393391928, + "num_tokens": 157890716.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.008814557393392, + "sampling/importance_sampling_ratio/mean": 0.9995736122131348, + "sampling/importance_sampling_ratio/min": 0.9439257462819417, + "sampling/sampling_logp_difference/max": 0.06689950758591294, + "sampling/sampling_logp_difference/mean": 0.001202967828915765, + "step": 12780, + "step_time": 4.594733424799051, + "student/entropy": 0.05389593628545602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.133333333333333, + "completions/max_terminated_length": 9.133333333333333, + "completions/mean_length": 1.1227778196334839, + "completions/mean_terminated_length": 1.1227778196334839, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06465975195169449, + "epoch": 0.48646185394751834, + "eval/gt_acc_batch": 0.0025000001614292463, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.785017967224121, + "kd/policy_loss": -6.810981341203054, + "kd/rkl_advantage_mean": 85.94079964955648, + "kd/rkl_advantage_p95": 107.80956312020619, + "kd/rkl_advantage_std": 37.93952339788278, + "kd/ssd_loss": 0.0, + "learning_rate": 5.135761212167242e-07, + "loss": -27.243924967447917, + "num_tokens": 158204094.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000000993410745, + "rewards/gt_logging_reward/std": 0.023483405510584515, + "sampling/importance_sampling_ratio/max": 1.1140775601069133, + "sampling/importance_sampling_ratio/mean": 1.000400084257126, + "sampling/importance_sampling_ratio/min": 0.9129073957602183, + "sampling/sampling_logp_difference/max": 0.14459194609274467, + "sampling/sampling_logp_difference/mean": 0.002269099282178407, + "step": 12810, + "step_time": 4.63995508121637, + "student/entropy": 0.06465975195169449 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.966666666666667, + "completions/max_terminated_length": 4.966666666666667, + "completions/mean_length": 1.0358333508173625, + "completions/mean_terminated_length": 1.0358333508173625, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03526086431617538, + "epoch": 0.4876011088747959, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.871282577514648, + "kd/policy_loss": 0.6054017265637716, + "kd/rkl_advantage_mean": 33.52895847956339, + "kd/rkl_advantage_p95": 50.47891351381938, + "kd/rkl_advantage_std": 25.99598462283611, + "kd/ssd_loss": 0.0, + "learning_rate": 5.124368662894467e-07, + "loss": 2.421605936686198, + "num_tokens": 158502423.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0218489487965903, + "sampling/importance_sampling_ratio/mean": 1.000054669380188, + "sampling/importance_sampling_ratio/min": 0.9761825044949849, + "sampling/sampling_logp_difference/max": 0.03889669796141485, + "sampling/sampling_logp_difference/mean": 0.000598219501262065, + "step": 12840, + "step_time": 4.625577454509524, + "student/entropy": 0.03526086431617538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.071666697661082, + "completions/mean_terminated_length": 1.071666697661082, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05824758956829707, + "epoch": 0.4887403638020734, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 51.50563049316406, + "kd/policy_loss": -2.3497689723968507, + "kd/rkl_advantage_mean": 46.04182696342468, + "kd/rkl_advantage_p95": 61.93282594680786, + "kd/rkl_advantage_std": 25.127117718259495, + "kd/ssd_loss": 0.0, + "learning_rate": 5.11297611362169e-07, + "loss": -9.399078369140625, + "num_tokens": 158801857.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.004285195469856262, + "sampling/importance_sampling_ratio/max": 1.0382295648256938, + "sampling/importance_sampling_ratio/mean": 1.0002476930618287, + "sampling/importance_sampling_ratio/min": 0.9619813919067383, + "sampling/sampling_logp_difference/max": 0.06948346705175937, + "sampling/sampling_logp_difference/mean": 0.0013286392514904341, + "step": 12870, + "step_time": 4.523542567148494, + "student/entropy": 0.05824758956829707 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.266666666666667, + "completions/max_terminated_length": 9.266666666666667, + "completions/mean_length": 1.1169444918632507, + "completions/mean_terminated_length": 1.1169444918632507, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06343286292006572, + "epoch": 0.489879618729351, + "eval/gt_acc_batch": 0.003055555714915196, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.78408145904541, + "kd/policy_loss": -6.5662723223368324, + "kd/rkl_advantage_mean": 89.83403034210205, + "kd/rkl_advantage_p95": 113.07104825973511, + "kd/rkl_advantage_std": 41.44211882154147, + "kd/ssd_loss": 0.0, + "learning_rate": 5.101583564348916e-07, + "loss": -26.265091959635416, + "num_tokens": 159119694.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.003055555714915196, + "rewards/gt_logging_reward/std": 0.02987071399887403, + "sampling/importance_sampling_ratio/max": 1.0646495819091797, + "sampling/importance_sampling_ratio/mean": 1.0001754760742188, + "sampling/importance_sampling_ratio/min": 0.9483968238035838, + "sampling/sampling_logp_difference/max": 0.08890234424422185, + "sampling/sampling_logp_difference/mean": 0.0016753989679273219, + "step": 12900, + "step_time": 4.612528776625792, + "student/entropy": 0.06343286292006572 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.0463889042536418, + "completions/mean_terminated_length": 1.0463889042536418, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.037946969348316394, + "epoch": 0.49101887365662855, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.515810012817383, + "kd/policy_loss": -0.1721242845058441, + "kd/rkl_advantage_mean": 43.770752636591595, + "kd/rkl_advantage_p95": 61.8422616481781, + "kd/rkl_advantage_std": 29.465454038977622, + "kd/ssd_loss": 0.0, + "learning_rate": 5.09019101507614e-07, + "loss": -0.6884989420572917, + "num_tokens": 159428941.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.028373654683431, + "sampling/importance_sampling_ratio/mean": 0.9999354759852092, + "sampling/importance_sampling_ratio/min": 0.9566893061002095, + "sampling/sampling_logp_difference/max": 0.06812120167693744, + "sampling/sampling_logp_difference/mean": 0.0009238410551915876, + "step": 12930, + "step_time": 4.538092390490541, + "student/entropy": 0.037946969348316394 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.333333333333333, + "completions/max_terminated_length": 6.333333333333333, + "completions/mean_length": 1.0761111378669739, + "completions/mean_terminated_length": 1.0761111378669739, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05372370583936572, + "epoch": 0.4921581285839061, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.297645568847656, + "kd/policy_loss": -2.0751395682493845, + "kd/rkl_advantage_mean": 49.531107346216835, + "kd/rkl_advantage_p95": 67.6872852007548, + "kd/rkl_advantage_std": 28.472332128882407, + "kd/ssd_loss": 0.0, + "learning_rate": 5.078798465803365e-07, + "loss": -8.30055898030599, + "num_tokens": 159747591.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0477786540985108, + "sampling/importance_sampling_ratio/mean": 0.9999089777469635, + "sampling/importance_sampling_ratio/min": 0.937078179915746, + "sampling/sampling_logp_difference/max": 0.07114139917927484, + "sampling/sampling_logp_difference/mean": 0.0013083682710809323, + "step": 12960, + "step_time": 4.477140067152989, + "student/entropy": 0.05372370583936572 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.566666666666666, + "completions/max_terminated_length": 8.566666666666666, + "completions/mean_length": 1.0886111418406168, + "completions/mean_terminated_length": 1.0886111418406168, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06208724075307449, + "epoch": 0.4932973835111837, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.11834716796875, + "kd/policy_loss": -3.271585921446482, + "kd/rkl_advantage_mean": 65.60491903622945, + "kd/rkl_advantage_p95": 84.76975820859273, + "kd/rkl_advantage_std": 33.41216141184171, + "kd/ssd_loss": 0.0, + "learning_rate": 5.067405916530588e-07, + "loss": -13.086343383789062, + "num_tokens": 160050486.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.070249851544698, + "sampling/importance_sampling_ratio/mean": 1.000317394733429, + "sampling/importance_sampling_ratio/min": 0.959189381202062, + "sampling/sampling_logp_difference/max": 0.11513462339838346, + "sampling/sampling_logp_difference/mean": 0.0018864477683867638, + "step": 12990, + "step_time": 4.6009060661541294, + "student/entropy": 0.06208724075307449 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.966666666666667, + "completions/max_terminated_length": 5.966666666666667, + "completions/mean_length": 1.0527777989705405, + "completions/mean_terminated_length": 1.0527777989705405, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05182686724389593, + "epoch": 0.49443663843846125, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.794437408447266, + "kd/policy_loss": -1.5159798502922057, + "kd/rkl_advantage_mean": 45.86805149714152, + "kd/rkl_advantage_p95": 62.058301496505734, + "kd/rkl_advantage_std": 27.087164122859637, + "kd/ssd_loss": 0.0, + "learning_rate": 5.056013367257813e-07, + "loss": -6.063921610514323, + "num_tokens": 160358460.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0636454423268635, + "sampling/importance_sampling_ratio/mean": 1.00031268397967, + "sampling/importance_sampling_ratio/min": 0.9673808415730795, + "sampling/sampling_logp_difference/max": 0.07740439131545523, + "sampling/sampling_logp_difference/mean": 0.0011628803690352167, + "step": 13020, + "step_time": 4.534049178881105, + "student/entropy": 0.05182686724389593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.233333333333333, + "completions/max_terminated_length": 9.233333333333333, + "completions/mean_length": 1.1155555963516235, + "completions/mean_terminated_length": 1.1155555963516235, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06985641429200769, + "epoch": 0.4955758933657388, + "eval/gt_acc_batch": 0.002222222338120143, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.646455764770508, + "kd/policy_loss": -5.022313551108042, + "kd/rkl_advantage_mean": 79.82708161671957, + "kd/rkl_advantage_p95": 101.76856621901194, + "kd/rkl_advantage_std": 38.40993520617485, + "kd/ssd_loss": 0.0, + "learning_rate": 5.044620817985038e-07, + "loss": -20.08925577799479, + "num_tokens": 160659284.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.002222222338120143, + "rewards/gt_logging_reward/std": 0.024343226353327432, + "sampling/importance_sampling_ratio/max": 1.0717591325441995, + "sampling/importance_sampling_ratio/mean": 1.0001066704591115, + "sampling/importance_sampling_ratio/min": 0.9259269098440807, + "sampling/sampling_logp_difference/max": 0.14153004556816692, + "sampling/sampling_logp_difference/mean": 0.002192740823375061, + "step": 13050, + "step_time": 4.608021531133757, + "student/entropy": 0.06985641429200769 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.533333333333333, + "completions/max_terminated_length": 7.533333333333333, + "completions/mean_length": 1.061388913790385, + "completions/mean_terminated_length": 1.061388913790385, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.036001341262211405, + "epoch": 0.4967151482930164, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.656253814697266, + "kd/policy_loss": -2.4615196188290915, + "kd/rkl_advantage_mean": 63.747113434473675, + "kd/rkl_advantage_p95": 84.68831033706665, + "kd/rkl_advantage_std": 36.068320735295615, + "kd/ssd_loss": 0.0, + "learning_rate": 5.033228268712261e-07, + "loss": -9.846077473958333, + "num_tokens": 160969561.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0364104072252909, + "sampling/importance_sampling_ratio/mean": 1.0000115394592286, + "sampling/importance_sampling_ratio/min": 0.9585505386193593, + "sampling/sampling_logp_difference/max": 0.07241351717772583, + "sampling/sampling_logp_difference/mean": 0.0009457654853273804, + "step": 13080, + "step_time": 4.676306866842788, + "student/entropy": 0.036001341262211405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.633333333333334, + "completions/max_terminated_length": 4.633333333333334, + "completions/mean_length": 1.035277791817983, + "completions/mean_terminated_length": 1.035277791817983, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04359901274243991, + "epoch": 0.49785440322029395, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.692827224731445, + "kd/policy_loss": 0.7121586720148723, + "kd/rkl_advantage_mean": 26.45002605120341, + "kd/rkl_advantage_p95": 39.384783061345416, + "kd/rkl_advantage_std": 20.29550128330787, + "kd/ssd_loss": 0.0, + "learning_rate": 5.021835719439487e-07, + "loss": 2.8486343383789063, + "num_tokens": 161278744.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0074573874473571, + "sampling/importance_sampling_ratio/mean": 0.9998451968034109, + "sampling/importance_sampling_ratio/min": 0.9622566024462382, + "sampling/sampling_logp_difference/max": 0.03916228274659564, + "sampling/sampling_logp_difference/mean": 0.0007039270276436582, + "step": 13110, + "step_time": 4.618012448342052, + "student/entropy": 0.04359901274243991 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 24.766666666666666, + "completions/max_terminated_length": 8.1, + "completions/mean_length": 1.2258333802223205, + "completions/mean_terminated_length": 1.0839146018028258, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06282203393056988, + "epoch": 0.49899365814757146, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.753787994384766, + "kd/policy_loss": -4.054366814096769, + "kd/rkl_advantage_mean": 75.08464190165202, + "kd/rkl_advantage_p95": 96.12724658648173, + "kd/rkl_advantage_std": 36.764541719357176, + "kd/ssd_loss": 0.0, + "learning_rate": 5.010443170166711e-07, + "loss": -16.217471313476562, + "num_tokens": 161591293.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.078990395863851, + "sampling/importance_sampling_ratio/mean": 0.9999344706535339, + "sampling/importance_sampling_ratio/min": 0.923026583592097, + "sampling/sampling_logp_difference/max": 0.10186995707141856, + "sampling/sampling_logp_difference/mean": 0.0015613102674251421, + "step": 13140, + "step_time": 4.581953177981389, + "student/entropy": 0.06282203393056988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.366666666666666, + "completions/max_terminated_length": 7.366666666666666, + "completions/mean_length": 1.1072222550710042, + "completions/mean_terminated_length": 1.1072222550710042, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06598308483759562, + "epoch": 0.5001329130748491, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 55.86509323120117, + "kd/policy_loss": -4.748693645000458, + "kd/rkl_advantage_mean": 63.037496423721315, + "kd/rkl_advantage_p95": 78.95973615646362, + "kd/rkl_advantage_std": 28.057910561561584, + "kd/ssd_loss": 0.0, + "learning_rate": 4.999050620893935e-07, + "loss": -18.994775390625, + "num_tokens": 161906623.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0512209296226502, + "sampling/importance_sampling_ratio/mean": 0.9999845604101817, + "sampling/importance_sampling_ratio/min": 0.9355706850687663, + "sampling/sampling_logp_difference/max": 0.10227541609977682, + "sampling/sampling_logp_difference/mean": 0.001856905678869225, + "step": 13170, + "step_time": 4.532525856397115, + "student/entropy": 0.06598308483759562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.0594444672266643, + "completions/mean_terminated_length": 1.0594444672266643, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05245458561306198, + "epoch": 0.5012721680021266, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.192373275756836, + "kd/policy_loss": -0.8329811890920004, + "kd/rkl_advantage_mean": 44.59970138867696, + "kd/rkl_advantage_p95": 62.31823189258576, + "kd/rkl_advantage_std": 28.268188921610513, + "kd/ssd_loss": 0.0, + "learning_rate": 4.98765807162116e-07, + "loss": -3.3319249471028645, + "num_tokens": 162203525.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0447771231333414, + "sampling/importance_sampling_ratio/mean": 1.0001653055349986, + "sampling/importance_sampling_ratio/min": 0.9673685789108276, + "sampling/sampling_logp_difference/max": 0.06249145135904352, + "sampling/sampling_logp_difference/mean": 0.0009516085498034954, + "step": 13200, + "step_time": 4.479218423471321, + "student/entropy": 0.05245458561306198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.3, + "completions/max_terminated_length": 6.3, + "completions/mean_length": 1.0641666928927103, + "completions/mean_terminated_length": 1.0641666928927103, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0612192607174317, + "epoch": 0.5024114229294042, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.921127319335938, + "kd/policy_loss": -1.872100087006887, + "kd/rkl_advantage_mean": 50.48682346343994, + "kd/rkl_advantage_p95": 67.24062838554383, + "kd/rkl_advantage_std": 28.897638003031414, + "kd/ssd_loss": 0.0, + "learning_rate": 4.976265522348384e-07, + "loss": -7.488400777180989, + "num_tokens": 162518916.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.005727223555247, + "sampling/importance_sampling_ratio/mean": 0.9995921611785888, + "sampling/importance_sampling_ratio/min": 0.9368318974971771, + "sampling/sampling_logp_difference/max": 0.06204164222969363, + "sampling/sampling_logp_difference/mean": 0.0011117068361879015, + "step": 13230, + "step_time": 4.5517074639365696, + "student/entropy": 0.0612192607174317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.633333333333334, + "completions/max_terminated_length": 7.633333333333334, + "completions/mean_length": 1.0675000309944154, + "completions/mean_terminated_length": 1.0675000309944154, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06763308616355061, + "epoch": 0.5035506778566817, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.12259864807129, + "kd/policy_loss": -3.19226549466451, + "kd/rkl_advantage_mean": 68.70883674621582, + "kd/rkl_advantage_p95": 90.61705448627472, + "kd/rkl_advantage_std": 37.745177874962486, + "kd/ssd_loss": 0.0, + "learning_rate": 4.964872973075608e-07, + "loss": -12.769059244791666, + "num_tokens": 162820351.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0387404680252075, + "sampling/importance_sampling_ratio/mean": 1.000007869799932, + "sampling/importance_sampling_ratio/min": 0.9387637416521708, + "sampling/sampling_logp_difference/max": 0.09748095315881074, + "sampling/sampling_logp_difference/mean": 0.0016062895078600074, + "step": 13260, + "step_time": 4.693245626089629, + "student/entropy": 0.06763308616355061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0575000246365864, + "completions/mean_terminated_length": 1.0575000246365864, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0467324895473818, + "epoch": 0.5046899327839592, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.99885082244873, + "kd/policy_loss": -1.9288191755612691, + "kd/rkl_advantage_mean": 50.96644760767619, + "kd/rkl_advantage_p95": 67.84910732110342, + "kd/rkl_advantage_std": 28.968254817525544, + "kd/ssd_loss": 0.0, + "learning_rate": 4.953480423802833e-07, + "loss": -7.715272521972656, + "num_tokens": 163128606.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0243799209594726, + "sampling/importance_sampling_ratio/mean": 0.9998412628968557, + "sampling/importance_sampling_ratio/min": 0.9439970274766286, + "sampling/sampling_logp_difference/max": 0.06333135237606863, + "sampling/sampling_logp_difference/mean": 0.0010783088073367253, + "step": 13290, + "step_time": 4.644106224307325, + "student/entropy": 0.0467324895473818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.166666666666666, + "completions/max_terminated_length": 8.166666666666666, + "completions/mean_length": 1.072777807712555, + "completions/mean_terminated_length": 1.072777807712555, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05450717384616534, + "epoch": 0.5058291877112369, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.693452835083008, + "kd/policy_loss": -3.160942852497101, + "kd/rkl_advantage_mean": 72.37074704170227, + "kd/rkl_advantage_p95": 96.29232212702433, + "kd/rkl_advantage_std": 40.18861284653346, + "kd/ssd_loss": 0.0, + "learning_rate": 4.942087874530057e-07, + "loss": -12.643772379557292, + "num_tokens": 163426100.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0108054081598918, + "sampling/importance_sampling_ratio/mean": 0.9996775428454081, + "sampling/importance_sampling_ratio/min": 0.9360509037971496, + "sampling/sampling_logp_difference/max": 0.07465041846347352, + "sampling/sampling_logp_difference/mean": 0.001212019108546277, + "step": 13320, + "step_time": 4.600899063124476, + "student/entropy": 0.05450717384616534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0700000246365866, + "completions/mean_terminated_length": 1.0700000246365866, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03986068712547421, + "epoch": 0.5069684426385144, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 29.76683235168457, + "kd/policy_loss": -2.571722880999247, + "kd/rkl_advantage_mean": 56.044411182403564, + "kd/rkl_advantage_p95": 73.2217878262202, + "kd/rkl_advantage_std": 29.832274535298346, + "kd/ssd_loss": 0.0, + "learning_rate": 4.930695325257282e-07, + "loss": -10.286891682942708, + "num_tokens": 163731496.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0281237800916037, + "sampling/importance_sampling_ratio/mean": 0.9997548341751099, + "sampling/importance_sampling_ratio/min": 0.942134286959966, + "sampling/sampling_logp_difference/max": 0.08406596926506608, + "sampling/sampling_logp_difference/mean": 0.0010265729026286863, + "step": 13350, + "step_time": 4.652937864023261, + "student/entropy": 0.03986068712547421 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.066666666666666, + "completions/max_terminated_length": 5.066666666666666, + "completions/mean_length": 1.0369444608688354, + "completions/mean_terminated_length": 1.0369444608688354, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04666447134998938, + "epoch": 0.508107697565792, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.951871871948242, + "kd/policy_loss": 0.23955742518107095, + "kd/rkl_advantage_mean": 34.43399146397908, + "kd/rkl_advantage_p95": 50.65314274628957, + "kd/rkl_advantage_std": 25.22054493824641, + "kd/ssd_loss": 0.0, + "learning_rate": 4.919302775984505e-07, + "loss": 0.9582309722900391, + "num_tokens": 164038189.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0550406376520793, + "sampling/importance_sampling_ratio/mean": 1.0003735939661662, + "sampling/importance_sampling_ratio/min": 0.9793312390645345, + "sampling/sampling_logp_difference/max": 0.05826233661888788, + "sampling/sampling_logp_difference/mean": 0.0008343126566614956, + "step": 13380, + "step_time": 4.627638945437502, + "student/entropy": 0.04666447134998938 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.433333333333334, + "completions/max_terminated_length": 7.433333333333334, + "completions/mean_length": 1.0963889201482138, + "completions/mean_terminated_length": 1.0963889201482138, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0693527903407812, + "epoch": 0.5092469524930695, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.450380325317383, + "kd/policy_loss": -3.267251245180766, + "kd/rkl_advantage_mean": 60.570776255925495, + "kd/rkl_advantage_p95": 79.08228143056233, + "kd/rkl_advantage_std": 31.27648055255413, + "kd/ssd_loss": 0.0, + "learning_rate": 4.90791022671173e-07, + "loss": -13.069004313151042, + "num_tokens": 164358384.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0834431250890095, + "sampling/importance_sampling_ratio/mean": 1.0004017174243927, + "sampling/importance_sampling_ratio/min": 0.9584170480569204, + "sampling/sampling_logp_difference/max": 0.07431150147070488, + "sampling/sampling_logp_difference/mean": 0.0015714443220834558, + "step": 13410, + "step_time": 4.705234976450447, + "student/entropy": 0.0693527903407812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.310344827586207, + "completions/max_terminated_length": 7.310344827586207, + "completions/mean_length": 1.0816092203403342, + "completions/mean_terminated_length": 1.0816092203403342, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.062057322822511196, + "epoch": 0.5103862074203471, + "eval/gt_acc_batch": 0.0017241380209552831, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.428823471069336, + "kd/policy_loss": -3.056433714669326, + "kd/rkl_advantage_mean": 55.43927791200835, + "kd/rkl_advantage_p95": 70.63169815622527, + "kd/rkl_advantage_std": 27.272694835375095, + "kd/ssd_loss": 0.0, + "learning_rate": 4.896517677438955e-07, + "loss": -11.818211873372396, + "num_tokens": 164650996.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0017241380209552831, + "rewards/gt_logging_reward/std": 0.017024284806744807, + "sampling/importance_sampling_ratio/max": 1.0067703641694168, + "sampling/importance_sampling_ratio/mean": 0.999677349781168, + "sampling/importance_sampling_ratio/min": 0.9480579228236757, + "sampling/sampling_logp_difference/max": 0.05964693556347027, + "sampling/sampling_logp_difference/mean": 0.0012450518502034888, + "step": 13440, + "step_time": 4.481569634579743, + "student/entropy": 0.062057322822511196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.5, + "completions/max_terminated_length": 7.5, + "completions/mean_length": 1.0794444719950358, + "completions/mean_terminated_length": 1.0794444719950358, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.055810808980216585, + "epoch": 0.5115254623476246, + "eval/gt_acc_batch": 0.0019444445768992105, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.09980297088623, + "kd/policy_loss": -3.640272398789724, + "kd/rkl_advantage_mean": 68.69571625391642, + "kd/rkl_advantage_p95": 90.30888399283091, + "kd/rkl_advantage_std": 36.79096654256185, + "kd/ssd_loss": 0.0, + "learning_rate": 4.885125128166179e-07, + "loss": -14.561092122395833, + "num_tokens": 164960042.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445148110389, + "rewards/gt_logging_reward/std": 0.017397598922252656, + "sampling/importance_sampling_ratio/max": 1.0562818805376688, + "sampling/importance_sampling_ratio/mean": 1.0004900534947714, + "sampling/importance_sampling_ratio/min": 0.9802742024262746, + "sampling/sampling_logp_difference/max": 0.06988658090898146, + "sampling/sampling_logp_difference/mean": 0.001350924865497897, + "step": 13470, + "step_time": 4.649635042762384, + "student/entropy": 0.055810808980216585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 23.033333333333335, + "completions/max_terminated_length": 6.333333333333333, + "completions/mean_length": 1.1869444807370504, + "completions/mean_terminated_length": 1.0450233658154806, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0471752910874784, + "epoch": 0.5126647172749023, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.07187843322754, + "kd/policy_loss": -0.7244374811649322, + "kd/rkl_advantage_mean": 52.114612849553424, + "kd/rkl_advantage_p95": 72.72592868010203, + "kd/rkl_advantage_std": 32.75787435968717, + "kd/ssd_loss": 0.0, + "learning_rate": 4.873732578893404e-07, + "loss": -2.8977516174316404, + "num_tokens": 165279411.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0498874107996623, + "sampling/importance_sampling_ratio/mean": 1.0000692745049795, + "sampling/importance_sampling_ratio/min": 0.949799237648646, + "sampling/sampling_logp_difference/max": 0.09844373858844241, + "sampling/sampling_logp_difference/mean": 0.0013111034272393833, + "step": 13500, + "step_time": 4.746485271681255, + "student/entropy": 0.0471752910874784 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.7, + "completions/max_terminated_length": 7.7, + "completions/mean_length": 1.0888889273007711, + "completions/mean_terminated_length": 1.0888889273007711, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06148768135656913, + "epoch": 0.5138039722021798, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 82.69377136230469, + "kd/policy_loss": -4.565575591723124, + "kd/rkl_advantage_mean": 69.15265207290649, + "kd/rkl_advantage_p95": 90.27451190948486, + "kd/rkl_advantage_std": 34.36891399423281, + "kd/ssd_loss": 0.0, + "learning_rate": 4.862340029620628e-07, + "loss": -18.26230265299479, + "num_tokens": 165580827.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0393328746159871, + "sampling/importance_sampling_ratio/mean": 1.0000992794831594, + "sampling/importance_sampling_ratio/min": 0.956004818280538, + "sampling/sampling_logp_difference/max": 0.07216870353246728, + "sampling/sampling_logp_difference/mean": 0.0011737315983433897, + "step": 13530, + "step_time": 4.540241062322942, + "student/entropy": 0.06148768135656913 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.966666666666667, + "completions/max_terminated_length": 5.966666666666667, + "completions/mean_length": 1.055833355585734, + "completions/mean_terminated_length": 1.055833355585734, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.045269291785856085, + "epoch": 0.5149432271294573, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.783602714538574, + "kd/policy_loss": -1.5412173668543498, + "kd/rkl_advantage_mean": 45.92019324302673, + "kd/rkl_advantage_p95": 62.05096657276154, + "kd/rkl_advantage_std": 26.907068412502607, + "kd/ssd_loss": 0.0, + "learning_rate": 4.850947480347852e-07, + "loss": -6.164869181315104, + "num_tokens": 165879916.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0307996193567912, + "sampling/importance_sampling_ratio/mean": 0.9998469889163971, + "sampling/importance_sampling_ratio/min": 0.9512002209822337, + "sampling/sampling_logp_difference/max": 0.060542526018495364, + "sampling/sampling_logp_difference/mean": 0.0009662966011092066, + "step": 13560, + "step_time": 4.68402014223005, + "student/entropy": 0.045269291785856085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.433333333333334, + "completions/max_terminated_length": 7.433333333333334, + "completions/mean_length": 1.0883333683013916, + "completions/mean_terminated_length": 1.0883333683013916, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04988775495439768, + "epoch": 0.5160824820567349, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 63.0365104675293, + "kd/policy_loss": -3.64339413245519, + "kd/rkl_advantage_mean": 64.25051043828329, + "kd/rkl_advantage_p95": 84.60879654884339, + "kd/rkl_advantage_std": 33.94517148335775, + "kd/ssd_loss": 0.0, + "learning_rate": 4.839554931075077e-07, + "loss": -14.573575846354167, + "num_tokens": 166192186.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0398086667060853, + "sampling/importance_sampling_ratio/mean": 1.0000965694586437, + "sampling/importance_sampling_ratio/min": 0.9513484875361125, + "sampling/sampling_logp_difference/max": 0.08867690378489594, + "sampling/sampling_logp_difference/mean": 0.0014159283717162907, + "step": 13590, + "step_time": 4.748148629437977, + "student/entropy": 0.04988775495439768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0608333587646483, + "completions/mean_terminated_length": 1.0608333587646483, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.046511078346520664, + "epoch": 0.5172217369840124, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.573258399963379, + "kd/policy_loss": -1.6185839315255484, + "kd/rkl_advantage_mean": 56.87085173924764, + "kd/rkl_advantage_p95": 78.88950839837392, + "kd/rkl_advantage_std": 35.13328246672948, + "kd/ssd_loss": 0.0, + "learning_rate": 4.828162381802301e-07, + "loss": -6.474335734049479, + "num_tokens": 166506277.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0258723298708599, + "sampling/importance_sampling_ratio/mean": 1.000070039431254, + "sampling/importance_sampling_ratio/min": 0.9681381106376648, + "sampling/sampling_logp_difference/max": 0.05922029193801184, + "sampling/sampling_logp_difference/mean": 0.0009237757854862139, + "step": 13620, + "step_time": 4.639465524132053, + "student/entropy": 0.046511078346520664 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.366666666666666, + "completions/max_terminated_length": 6.366666666666666, + "completions/mean_length": 1.0516666889190673, + "completions/mean_terminated_length": 1.0516666889190673, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.045801228595276675, + "epoch": 0.51836099191129, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.803478240966797, + "kd/policy_loss": -1.6222981135050456, + "kd/rkl_advantage_mean": 53.54765775998433, + "kd/rkl_advantage_p95": 73.5058121919632, + "kd/rkl_advantage_std": 32.96321199735006, + "kd/ssd_loss": 0.0, + "learning_rate": 4.816769832529525e-07, + "loss": -6.4891916910807295, + "num_tokens": 166807247.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0308021823565165, + "sampling/importance_sampling_ratio/mean": 1.0000153044859568, + "sampling/importance_sampling_ratio/min": 0.96045703291893, + "sampling/sampling_logp_difference/max": 0.07432829129199187, + "sampling/sampling_logp_difference/mean": 0.001005855269128612, + "step": 13650, + "step_time": 4.551728829574616, + "student/entropy": 0.045801228595276675 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.0677778045336406, + "completions/mean_terminated_length": 1.0677778045336406, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05228769136592746, + "epoch": 0.5195002468385675, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.89558219909668, + "kd/policy_loss": -2.8041027506192524, + "kd/rkl_advantage_mean": 47.86652189890544, + "kd/rkl_advantage_p95": 62.46609259446462, + "kd/rkl_advantage_std": 24.051517160733543, + "kd/ssd_loss": 0.0, + "learning_rate": 4.80537728325675e-07, + "loss": -11.216412353515626, + "num_tokens": 167113859.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0874091108640036, + "sampling/importance_sampling_ratio/mean": 1.0005198458830515, + "sampling/importance_sampling_ratio/min": 0.9643096764882405, + "sampling/sampling_logp_difference/max": 0.07998646216777464, + "sampling/sampling_logp_difference/mean": 0.0013194836055239042, + "step": 13680, + "step_time": 4.605112570516455, + "student/entropy": 0.05228769136592746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.8, + "completions/max_terminated_length": 7.8, + "completions/mean_length": 1.0988889296849569, + "completions/mean_terminated_length": 1.0988889296849569, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054225388417641324, + "epoch": 0.5206395017658452, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.187829971313477, + "kd/policy_loss": -4.957050160566966, + "kd/rkl_advantage_mean": 71.41026040712993, + "kd/rkl_advantage_p95": 90.18668525218963, + "kd/rkl_advantage_std": 33.454289176066716, + "kd/ssd_loss": 0.0, + "learning_rate": 4.793984733983975e-07, + "loss": -19.82820027669271, + "num_tokens": 167411527.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.035505755742391, + "sampling/importance_sampling_ratio/mean": 0.9997198224067688, + "sampling/importance_sampling_ratio/min": 0.9367703219254812, + "sampling/sampling_logp_difference/max": 0.08679052363149822, + "sampling/sampling_logp_difference/mean": 0.0016148706956300885, + "step": 13710, + "step_time": 4.648555510215616, + "student/entropy": 0.054225388417641324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.166666666666667, + "completions/max_terminated_length": 6.166666666666667, + "completions/mean_length": 1.0538889129956563, + "completions/mean_terminated_length": 1.0538889129956563, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04385538538917899, + "epoch": 0.5217787566931227, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.962063789367676, + "kd/policy_loss": -1.5465396046638489, + "kd/rkl_advantage_mean": 49.146015135447186, + "kd/rkl_advantage_p95": 67.31768271923065, + "kd/rkl_advantage_std": 29.88917339841525, + "kd/ssd_loss": 0.0, + "learning_rate": 4.782592184711198e-07, + "loss": -6.186159261067709, + "num_tokens": 167731425.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.024464452266693, + "sampling/importance_sampling_ratio/mean": 1.0000030815601348, + "sampling/importance_sampling_ratio/min": 0.9579295992851258, + "sampling/sampling_logp_difference/max": 0.06769397988294562, + "sampling/sampling_logp_difference/mean": 0.001144830894190818, + "step": 13740, + "step_time": 4.716019489806301, + "student/entropy": 0.04385538538917899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.8, + "completions/max_terminated_length": 6.8, + "completions/mean_length": 1.0583333571751912, + "completions/mean_terminated_length": 1.0583333571751912, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.042796181080241996, + "epoch": 0.5229180116204003, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.173330307006836, + "kd/policy_loss": -1.345267109076182, + "kd/rkl_advantage_mean": 53.46134306589762, + "kd/rkl_advantage_p95": 73.31777886549632, + "kd/rkl_advantage_std": 32.77192853639523, + "kd/ssd_loss": 0.0, + "learning_rate": 4.771199635438423e-07, + "loss": -5.3810679117838545, + "num_tokens": 168035603.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0062025705973308, + "sampling/importance_sampling_ratio/mean": 0.9997941215833028, + "sampling/importance_sampling_ratio/min": 0.9588302950064341, + "sampling/sampling_logp_difference/max": 0.0644539267445604, + "sampling/sampling_logp_difference/mean": 0.0008541622514409634, + "step": 13770, + "step_time": 4.677382144708342, + "student/entropy": 0.042796181080241996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.1, + "completions/max_terminated_length": 5.1, + "completions/mean_length": 1.0500000238418579, + "completions/mean_terminated_length": 1.0500000238418579, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05245519295955698, + "epoch": 0.5240572665476778, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.602258682250977, + "kd/policy_loss": -0.38052963813145957, + "kd/rkl_advantage_mean": 35.66235629717509, + "kd/rkl_advantage_p95": 50.50452448527018, + "kd/rkl_advantage_std": 24.05901886423429, + "kd/ssd_loss": 0.0, + "learning_rate": 4.7598070861656474e-07, + "loss": -1.522119903564453, + "num_tokens": 168350807.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0344514091809591, + "sampling/importance_sampling_ratio/mean": 1.000285836060842, + "sampling/importance_sampling_ratio/min": 0.9839972039063771, + "sampling/sampling_logp_difference/max": 0.05221033080791434, + "sampling/sampling_logp_difference/mean": 0.0009745399700477719, + "step": 13800, + "step_time": 4.692363179370295, + "student/entropy": 0.05245519295955698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.866666666666667, + "completions/max_terminated_length": 9.866666666666667, + "completions/mean_length": 1.1269444863001505, + "completions/mean_terminated_length": 1.1269444863001505, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06325278148675959, + "epoch": 0.5251965214749553, + "eval/gt_acc_batch": 0.0025000001303851606, + "frac_reward_zero_std": 1.0, + "grad_norm": 42.16261291503906, + "kd/policy_loss": -6.708260631561279, + "kd/rkl_advantage_mean": 90.28365306854248, + "kd/rkl_advantage_p95": 113.42355150381724, + "kd/rkl_advantage_std": 40.6847105105718, + "kd/ssd_loss": 0.0, + "learning_rate": 4.748414536892872e-07, + "loss": -26.833052571614584, + "num_tokens": 168659920.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000001303851606, + "rewards/gt_logging_reward/std": 0.02378490741054217, + "sampling/importance_sampling_ratio/max": 1.0573589086532593, + "sampling/importance_sampling_ratio/mean": 0.9999093492825826, + "sampling/importance_sampling_ratio/min": 0.919810297091802, + "sampling/sampling_logp_difference/max": 0.12113069106514256, + "sampling/sampling_logp_difference/mean": 0.0019592751409315194, + "step": 13830, + "step_time": 4.713152930346163, + "student/entropy": 0.06325278148675959 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.066111139456431, + "completions/mean_terminated_length": 1.066111139456431, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.039119493883724016, + "epoch": 0.526335776402233, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.746490478515625, + "kd/policy_loss": -2.26556959549586, + "kd/rkl_advantage_mean": 58.57242126464844, + "kd/rkl_advantage_p95": 79.2066026687622, + "kd/rkl_advantage_std": 33.98124744693438, + "kd/ssd_loss": 0.0, + "learning_rate": 4.737021987620096e-07, + "loss": -9.0622802734375, + "num_tokens": 168964678.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0323180079460144, + "sampling/importance_sampling_ratio/mean": 1.0000731388727824, + "sampling/importance_sampling_ratio/min": 0.965347683429718, + "sampling/sampling_logp_difference/max": 0.05866848255197207, + "sampling/sampling_logp_difference/mean": 0.0008770103600303021, + "step": 13860, + "step_time": 4.676310639432631, + "student/entropy": 0.039119493883724016 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.1, + "completions/max_terminated_length": 5.1, + "completions/mean_length": 1.0472222447395325, + "completions/mean_terminated_length": 1.0472222447395325, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04305555721124013, + "epoch": 0.5274750313295105, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.914840698242188, + "kd/policy_loss": -0.07354604601860046, + "kd/rkl_advantage_mean": 35.29450380007426, + "kd/rkl_advantage_p95": 50.73663864135742, + "kd/rkl_advantage_std": 24.571179988980294, + "kd/ssd_loss": 0.0, + "learning_rate": 4.7256294383473204e-07, + "loss": -0.2941843032836914, + "num_tokens": 169276288.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0067999203999838, + "sampling/importance_sampling_ratio/mean": 0.9996883710225423, + "sampling/importance_sampling_ratio/min": 0.955447667837143, + "sampling/sampling_logp_difference/max": 0.0495227131449307, + "sampling/sampling_logp_difference/mean": 0.0008296342091246819, + "step": 13890, + "step_time": 4.573856754729059, + "student/entropy": 0.04305555721124013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.033333333333333, + "completions/max_terminated_length": 8.033333333333333, + "completions/mean_length": 1.079722253481547, + "completions/mean_terminated_length": 1.079722253481547, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049906638202567896, + "epoch": 0.5286142862567881, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.180477142333984, + "kd/policy_loss": -4.438185067971547, + "kd/rkl_advantage_mean": 74.7185698668162, + "kd/rkl_advantage_p95": 96.25963746706644, + "kd/rkl_advantage_std": 37.91641823848089, + "kd/ssd_loss": 0.0, + "learning_rate": 4.7142368890745446e-07, + "loss": -17.752742513020834, + "num_tokens": 169597463.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0291351556777955, + "sampling/importance_sampling_ratio/mean": 0.9998548666636149, + "sampling/importance_sampling_ratio/min": 0.9382082223892212, + "sampling/sampling_logp_difference/max": 0.08667663984621564, + "sampling/sampling_logp_difference/mean": 0.0013951345409926339, + "step": 13920, + "step_time": 4.767581189086195, + "student/entropy": 0.049906638202567896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.966666666666667, + "completions/max_terminated_length": 6.966666666666667, + "completions/mean_length": 1.0658333579699197, + "completions/mean_terminated_length": 1.0658333579699197, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0456682442377011, + "epoch": 0.5297535411840656, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.973261833190918, + "kd/policy_loss": -2.273424498240153, + "kd/rkl_advantage_mean": 55.448095846176145, + "kd/rkl_advantage_p95": 73.27369739214579, + "kd/rkl_advantage_std": 30.545144126812616, + "kd/ssd_loss": 0.0, + "learning_rate": 4.7028443398017693e-07, + "loss": -9.093699137369791, + "num_tokens": 169894108.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0197578112284342, + "sampling/importance_sampling_ratio/mean": 0.9997796555360158, + "sampling/importance_sampling_ratio/min": 0.9468866606553396, + "sampling/sampling_logp_difference/max": 0.07479072681938609, + "sampling/sampling_logp_difference/mean": 0.0010760256031062455, + "step": 13950, + "step_time": 4.665317149068384, + "student/entropy": 0.0456682442377011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.733333333333333, + "completions/max_terminated_length": 6.733333333333333, + "completions/mean_length": 1.061666691303253, + "completions/mean_terminated_length": 1.061666691303253, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05059886043891311, + "epoch": 0.5308927961113432, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.799562454223633, + "kd/policy_loss": -2.258545768260956, + "kd/rkl_advantage_mean": 54.90139284133911, + "kd/rkl_advantage_p95": 73.09096318880717, + "kd/rkl_advantage_std": 30.779396852850915, + "kd/ssd_loss": 0.0, + "learning_rate": 4.691451790528994e-07, + "loss": -9.034184773763021, + "num_tokens": 170206834.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0269703149795533, + "sampling/importance_sampling_ratio/mean": 0.9997573077678681, + "sampling/importance_sampling_ratio/min": 0.9440800706545512, + "sampling/sampling_logp_difference/max": 0.0872595375093321, + "sampling/sampling_logp_difference/mean": 0.001429981984741365, + "step": 13980, + "step_time": 4.7733385280434355, + "student/entropy": 0.05059886043891311 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.733333333333333, + "completions/max_terminated_length": 6.733333333333333, + "completions/mean_length": 1.061666695276896, + "completions/mean_terminated_length": 1.061666695276896, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052927059959620236, + "epoch": 0.5320320510386207, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.830095291137695, + "kd/policy_loss": -2.522236172358195, + "kd/rkl_advantage_mean": 57.917357444763184, + "kd/rkl_advantage_p95": 78.67686174710592, + "kd/rkl_advantage_std": 33.29171518782775, + "kd/ssd_loss": 0.0, + "learning_rate": 4.680059241256218e-07, + "loss": -10.088946533203124, + "num_tokens": 170511088.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.018820861975352, + "sampling/importance_sampling_ratio/mean": 0.9999141792456309, + "sampling/importance_sampling_ratio/min": 0.9625404834747314, + "sampling/sampling_logp_difference/max": 0.04971645992870132, + "sampling/sampling_logp_difference/mean": 0.0009584945306414739, + "step": 14010, + "step_time": 4.668000155155702, + "student/entropy": 0.052927059959620236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.833333333333333, + "completions/max_terminated_length": 6.833333333333333, + "completions/mean_length": 1.0563889145851135, + "completions/mean_terminated_length": 1.0563889145851135, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04208610064039628, + "epoch": 0.5331713059658983, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 30.443613052368164, + "kd/policy_loss": -2.3733158310254416, + "kd/rkl_advantage_mean": 58.995837704340616, + "kd/rkl_advantage_p95": 78.94637014865876, + "kd/rkl_advantage_std": 34.01250504752, + "kd/ssd_loss": 0.0, + "learning_rate": 4.668666691983443e-07, + "loss": -9.49326171875, + "num_tokens": 170819667.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.037958562374115, + "sampling/importance_sampling_ratio/mean": 0.9999895393848419, + "sampling/importance_sampling_ratio/min": 0.9533239861329397, + "sampling/sampling_logp_difference/max": 0.07723350802746912, + "sampling/sampling_logp_difference/mean": 0.0011504234978929162, + "step": 14040, + "step_time": 4.491334783297498, + "student/entropy": 0.04208610064039628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.666666666666667, + "completions/max_terminated_length": 5.666666666666667, + "completions/mean_length": 1.0572222431500753, + "completions/mean_terminated_length": 1.0572222431500753, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0377199072856456, + "epoch": 0.5343105608931759, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 36.936744689941406, + "kd/policy_loss": -1.0120111982027689, + "kd/rkl_advantage_mean": 41.269627300898236, + "kd/rkl_advantage_p95": 56.41476457118988, + "kd/rkl_advantage_std": 25.36664245724678, + "kd/ssd_loss": 0.0, + "learning_rate": 4.657274142710667e-07, + "loss": -4.048044840494792, + "num_tokens": 171123401.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0284996231396992, + "sampling/importance_sampling_ratio/mean": 0.9998630066712697, + "sampling/importance_sampling_ratio/min": 0.9552366852760314, + "sampling/sampling_logp_difference/max": 0.06810597539103279, + "sampling/sampling_logp_difference/mean": 0.000984966047205186, + "step": 14070, + "step_time": 4.508638690089962, + "student/entropy": 0.0377199072856456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.066666666666666, + "completions/max_terminated_length": 6.066666666666666, + "completions/mean_length": 1.0586111386617025, + "completions/mean_terminated_length": 1.0586111386617025, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04613351595277588, + "epoch": 0.5354498158204535, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 206.19308471679688, + "kd/policy_loss": -2.0728466331958773, + "kd/rkl_advantage_mean": 48.679471747080484, + "kd/rkl_advantage_p95": 67.46965206464132, + "kd/rkl_advantage_std": 28.531584268808366, + "kd/ssd_loss": 0.0, + "learning_rate": 4.645881593437891e-07, + "loss": -8.291387430826823, + "num_tokens": 171428972.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0243716518084207, + "sampling/importance_sampling_ratio/mean": 0.9999244014422098, + "sampling/importance_sampling_ratio/min": 0.9590064436197281, + "sampling/sampling_logp_difference/max": 0.0750014858863627, + "sampling/sampling_logp_difference/mean": 0.0010923685496284938, + "step": 14100, + "step_time": 4.526735331913612, + "student/entropy": 0.04613351595277588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.072777803738912, + "completions/mean_terminated_length": 1.072777803738912, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05267508790517847, + "epoch": 0.536589070747731, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.923521995544434, + "kd/policy_loss": -2.8231292804082235, + "kd/rkl_advantage_mean": 58.500715764363605, + "kd/rkl_advantage_p95": 79.22069209416708, + "kd/rkl_advantage_std": 32.69515669544538, + "kd/ssd_loss": 0.0, + "learning_rate": 4.634489044165116e-07, + "loss": -11.292519124348958, + "num_tokens": 171744922.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0110714515050252, + "sampling/importance_sampling_ratio/mean": 0.9993981103102366, + "sampling/importance_sampling_ratio/min": 0.926251353820165, + "sampling/sampling_logp_difference/max": 0.07659070906229318, + "sampling/sampling_logp_difference/mean": 0.0013472674133178468, + "step": 14130, + "step_time": 4.504696939862333, + "student/entropy": 0.05267508790517847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.2, + "completions/max_terminated_length": 7.2, + "completions/mean_length": 1.078611143430074, + "completions/mean_terminated_length": 1.078611143430074, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05252367152522008, + "epoch": 0.5377283256750085, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.327798843383789, + "kd/policy_loss": -4.239789275328318, + "kd/rkl_advantage_mean": 64.64455609321594, + "kd/rkl_advantage_p95": 84.72662180264791, + "kd/rkl_advantage_std": 32.16651467084885, + "kd/ssd_loss": 0.0, + "learning_rate": 4.62309649489234e-07, + "loss": -16.959157307942707, + "num_tokens": 172040885.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.040609057744344, + "sampling/importance_sampling_ratio/mean": 1.0001300732294718, + "sampling/importance_sampling_ratio/min": 0.9631085495154063, + "sampling/sampling_logp_difference/max": 0.08579140012152493, + "sampling/sampling_logp_difference/mean": 0.0013289337415092934, + "step": 14160, + "step_time": 4.4254641613340935, + "student/entropy": 0.05252367152522008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.066666666666666, + "completions/max_terminated_length": 7.066666666666666, + "completions/mean_length": 1.0738889217376708, + "completions/mean_terminated_length": 1.0738889217376708, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03953364045980076, + "epoch": 0.5388675806022861, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.332216262817383, + "kd/policy_loss": -2.6156384984652203, + "kd/rkl_advantage_mean": 59.764233859380084, + "kd/rkl_advantage_p95": 79.17887634436289, + "kd/rkl_advantage_std": 32.9567493836085, + "kd/ssd_loss": 0.0, + "learning_rate": 4.6117039456195643e-07, + "loss": -10.462552897135417, + "num_tokens": 172354927.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.047865056991577, + "sampling/importance_sampling_ratio/mean": 1.0000003417332968, + "sampling/importance_sampling_ratio/min": 0.9418975174427032, + "sampling/sampling_logp_difference/max": 0.08830067373346537, + "sampling/sampling_logp_difference/mean": 0.0012800855760967049, + "step": 14190, + "step_time": 4.460622670183269, + "student/entropy": 0.03953364045980076 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.666666666666667, + "completions/max_terminated_length": 6.666666666666667, + "completions/mean_length": 1.0669444719950358, + "completions/mean_terminated_length": 1.0669444719950358, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04385932898148894, + "epoch": 0.5400068355295636, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.696651458740234, + "kd/policy_loss": -2.245714811484019, + "kd/rkl_advantage_mean": 55.05304770469665, + "kd/rkl_advantage_p95": 73.6592973391215, + "kd/rkl_advantage_std": 31.1363022228082, + "kd/ssd_loss": 0.0, + "learning_rate": 4.600311396346789e-07, + "loss": -8.982859293619791, + "num_tokens": 172668392.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0552464644114177, + "sampling/importance_sampling_ratio/mean": 1.0003439823786417, + "sampling/importance_sampling_ratio/min": 0.9716513276100158, + "sampling/sampling_logp_difference/max": 0.06230455130183448, + "sampling/sampling_logp_difference/mean": 0.0009501444466877729, + "step": 14220, + "step_time": 4.5164170022976275, + "student/entropy": 0.04385932898148894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.075000031789144, + "completions/mean_terminated_length": 1.075000031789144, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04407578983033697, + "epoch": 0.5411460904568413, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.273484230041504, + "kd/policy_loss": -3.6686811129252117, + "kd/rkl_advantage_mean": 65.07447824478149, + "kd/rkl_advantage_p95": 84.7631494919459, + "kd/rkl_advantage_std": 33.58229520916939, + "kd/ssd_loss": 0.0, + "learning_rate": 4.588918847074013e-07, + "loss": -14.674725341796876, + "num_tokens": 172970550.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0142706394195558, + "sampling/importance_sampling_ratio/mean": 0.9996983269850414, + "sampling/importance_sampling_ratio/min": 0.9526082277297974, + "sampling/sampling_logp_difference/max": 0.06757531793943296, + "sampling/sampling_logp_difference/mean": 0.0012069645978044718, + "step": 14250, + "step_time": 4.347032794286497, + "student/entropy": 0.04407578983033697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.1, + "completions/max_terminated_length": 5.1, + "completions/mean_length": 1.045555571715037, + "completions/mean_terminated_length": 1.045555571715037, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04120707850282391, + "epoch": 0.5422853453841188, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.78977108001709, + "kd/policy_loss": -0.37236781120300294, + "kd/rkl_advantage_mean": 32.34917149543762, + "kd/rkl_advantage_p95": 45.239102300008135, + "kd/rkl_advantage_std": 20.82385650674502, + "kd/ssd_loss": 0.0, + "learning_rate": 4.577526297801238e-07, + "loss": -1.4894718170166015, + "num_tokens": 173276970.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0229862014452615, + "sampling/importance_sampling_ratio/mean": 0.9998861471811931, + "sampling/importance_sampling_ratio/min": 0.9538359045982361, + "sampling/sampling_logp_difference/max": 0.0653557694517076, + "sampling/sampling_logp_difference/mean": 0.0009508474826967965, + "step": 14280, + "step_time": 4.188437555322889, + "student/entropy": 0.04120707850282391 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0816666960716248, + "completions/mean_terminated_length": 1.0816666960716248, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06072546625509858, + "epoch": 0.5434246003113964, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 28.489452362060547, + "kd/policy_loss": -3.1392242391904195, + "kd/rkl_advantage_mean": 56.51665755907695, + "kd/rkl_advantage_p95": 73.08418854077657, + "kd/rkl_advantage_std": 28.383038751284282, + "kd/ssd_loss": 0.0, + "learning_rate": 4.566133748528462e-07, + "loss": -12.556895955403645, + "num_tokens": 173584464.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0809404293696085, + "sampling/importance_sampling_ratio/mean": 1.0004393815994264, + "sampling/importance_sampling_ratio/min": 0.956750609477361, + "sampling/sampling_logp_difference/max": 0.09226229252914588, + "sampling/sampling_logp_difference/mean": 0.0015876754943747074, + "step": 14310, + "step_time": 4.259656735738584, + "student/entropy": 0.06072546625509858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.3, + "completions/max_terminated_length": 8.3, + "completions/mean_length": 1.0805555860201517, + "completions/mean_terminated_length": 1.0805555860201517, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.059454323071986434, + "epoch": 0.5445638552386739, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.404479026794434, + "kd/policy_loss": -3.8121929208437604, + "kd/rkl_advantage_mean": 73.65761698087057, + "kd/rkl_advantage_p95": 96.2935527642568, + "kd/rkl_advantage_std": 38.70372229417165, + "kd/ssd_loss": 0.0, + "learning_rate": 4.554741199255687e-07, + "loss": -15.248771158854167, + "num_tokens": 173886322.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0202311873435974, + "sampling/importance_sampling_ratio/mean": 0.999634850025177, + "sampling/importance_sampling_ratio/min": 0.927789298693339, + "sampling/sampling_logp_difference/max": 0.09136937512084842, + "sampling/sampling_logp_difference/mean": 0.0014927472851316754, + "step": 14340, + "step_time": 4.237702580378391, + "student/entropy": 0.059454323071986434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.0452777981758117, + "completions/mean_terminated_length": 1.0452777981758117, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0436323506757617, + "epoch": 0.5457031101659515, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.76577377319336, + "kd/policy_loss": -0.9692310531934102, + "kd/rkl_advantage_mean": 48.85133434931437, + "kd/rkl_advantage_p95": 67.9505695104599, + "kd/rkl_advantage_std": 31.432316700617474, + "kd/ssd_loss": 0.0, + "learning_rate": 4.543348649982911e-07, + "loss": -3.8769241333007813, + "num_tokens": 174187973.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.001452334721883, + "sampling/importance_sampling_ratio/mean": 0.999683811267217, + "sampling/importance_sampling_ratio/min": 0.9544668614864349, + "sampling/sampling_logp_difference/max": 0.047394569497555494, + "sampling/sampling_logp_difference/mean": 0.0008020154641902384, + "step": 14370, + "step_time": 4.2827089019080935, + "student/entropy": 0.0436323506757617 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.266666666666667, + "completions/max_terminated_length": 5.266666666666667, + "completions/mean_length": 1.0577777942021689, + "completions/mean_terminated_length": 1.0577777942021689, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05098875450591246, + "epoch": 0.546842365093229, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.08367156982422, + "kd/policy_loss": -0.15214047829310098, + "kd/rkl_advantage_mean": 32.21438460350036, + "kd/rkl_advantage_p95": 45.18455770810445, + "kd/rkl_advantage_std": 21.16190446416537, + "kd/ssd_loss": 0.0, + "learning_rate": 4.531956100710135e-07, + "loss": -0.6085629145304362, + "num_tokens": 174495957.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.035115691026052, + "sampling/importance_sampling_ratio/mean": 1.0001077155272167, + "sampling/importance_sampling_ratio/min": 0.9674944619337718, + "sampling/sampling_logp_difference/max": 0.05464027232180039, + "sampling/sampling_logp_difference/mean": 0.001049745481577702, + "step": 14400, + "step_time": 4.4296691660226015, + "student/entropy": 0.05098875450591246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.633333333333334, + "completions/max_terminated_length": 7.633333333333334, + "completions/mean_length": 1.0913889288902283, + "completions/mean_terminated_length": 1.0913889288902283, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06642961561058959, + "epoch": 0.5479816200205065, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.81223487854004, + "kd/policy_loss": -4.58838074405988, + "kd/rkl_advantage_mean": 69.22288157145182, + "kd/rkl_advantage_p95": 90.72470965385438, + "kd/rkl_advantage_std": 34.670733934640886, + "kd/ssd_loss": 0.0, + "learning_rate": 4.52056355143736e-07, + "loss": -18.353523763020835, + "num_tokens": 174808318.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0553393681844077, + "sampling/importance_sampling_ratio/mean": 0.9998636623223622, + "sampling/importance_sampling_ratio/min": 0.9349848330020905, + "sampling/sampling_logp_difference/max": 0.09176185221100847, + "sampling/sampling_logp_difference/mean": 0.0015106973490522553, + "step": 14430, + "step_time": 4.568026963854209, + "student/entropy": 0.06642961561058959 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.0688889145851135, + "completions/mean_terminated_length": 1.0688889145851135, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04798623540749152, + "epoch": 0.5491208749477842, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.015937805175781, + "kd/policy_loss": -2.374182653427124, + "kd/rkl_advantage_mean": 59.508572800954184, + "kd/rkl_advantage_p95": 79.41137998104095, + "kd/rkl_advantage_std": 33.363619855046274, + "kd/ssd_loss": 0.0, + "learning_rate": 4.509171002164584e-07, + "loss": -9.496732584635417, + "num_tokens": 175117430.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.007937753200531, + "sampling/importance_sampling_ratio/mean": 0.999484407901764, + "sampling/importance_sampling_ratio/min": 0.9242332379023234, + "sampling/sampling_logp_difference/max": 0.0850199459043021, + "sampling/sampling_logp_difference/mean": 0.0011551396164577454, + "step": 14460, + "step_time": 4.487922556430567, + "student/entropy": 0.04798623540749152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.866666666666666, + "completions/max_terminated_length": 5.866666666666666, + "completions/mean_length": 1.0522222479184469, + "completions/mean_terminated_length": 1.0522222479184469, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0440562619206806, + "epoch": 0.5502601298750617, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.78252410888672, + "kd/policy_loss": -1.435412440697352, + "kd/rkl_advantage_mean": 47.6506210009257, + "kd/rkl_advantage_p95": 67.25633300145468, + "kd/rkl_advantage_std": 30.313512551784516, + "kd/ssd_loss": 0.0, + "learning_rate": 4.497778452891808e-07, + "loss": -5.741649373372396, + "num_tokens": 175433882.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.012066396077474, + "sampling/importance_sampling_ratio/mean": 0.9998199204603831, + "sampling/importance_sampling_ratio/min": 0.9714175860087076, + "sampling/sampling_logp_difference/max": 0.05322397644631564, + "sampling/sampling_logp_difference/mean": 0.0008950646088730234, + "step": 14490, + "step_time": 4.421807000852035, + "student/entropy": 0.0440562619206806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.2, + "completions/max_terminated_length": 8.2, + "completions/mean_length": 1.084166701634725, + "completions/mean_terminated_length": 1.084166701634725, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04492248181874554, + "epoch": 0.5513993848023393, + "eval/gt_acc_batch": 0.0025000001303851606, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.577397346496582, + "kd/policy_loss": -4.51287016471227, + "kd/rkl_advantage_mean": 75.35378166834514, + "kd/rkl_advantage_p95": 96.58217425346375, + "kd/rkl_advantage_std": 37.58975327412288, + "kd/ssd_loss": 0.0, + "learning_rate": 4.486385903619033e-07, + "loss": -18.051483154296875, + "num_tokens": 175741689.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000001303851606, + "rewards/gt_logging_reward/std": 0.02378490741054217, + "sampling/importance_sampling_ratio/max": 1.0320700844128927, + "sampling/importance_sampling_ratio/mean": 0.9999196191628774, + "sampling/importance_sampling_ratio/min": 0.955525420109431, + "sampling/sampling_logp_difference/max": 0.08395498073659838, + "sampling/sampling_logp_difference/mean": 0.0011200993384894295, + "step": 14520, + "step_time": 4.434865027817432, + "student/entropy": 0.04492248181874554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.233333333333333, + "completions/max_terminated_length": 6.233333333333333, + "completions/mean_length": 1.0583333571751912, + "completions/mean_terminated_length": 1.0583333571751912, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.038731198540578285, + "epoch": 0.5525386397296168, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.847366333007812, + "kd/policy_loss": -1.2473120888074238, + "kd/rkl_advantage_mean": 49.063235410054524, + "kd/rkl_advantage_p95": 67.77849535942077, + "kd/rkl_advantage_std": 30.4042890638113, + "kd/ssd_loss": 0.0, + "learning_rate": 4.4749933543462576e-07, + "loss": -4.989247639973958, + "num_tokens": 176046131.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0629709959030151, + "sampling/importance_sampling_ratio/mean": 1.0001959721247355, + "sampling/importance_sampling_ratio/min": 0.9587162653605144, + "sampling/sampling_logp_difference/max": 0.07265931254563232, + "sampling/sampling_logp_difference/mean": 0.0010960845262161455, + "step": 14550, + "step_time": 4.343433477690754, + "student/entropy": 0.038731198540578285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.5, + "completions/max_terminated_length": 7.5, + "completions/mean_length": 1.0794444759686788, + "completions/mean_terminated_length": 1.0794444759686788, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04743657391518354, + "epoch": 0.5536778946568944, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.496217727661133, + "kd/policy_loss": -3.6930405537287396, + "kd/rkl_advantage_mean": 65.48162016868591, + "kd/rkl_advantage_p95": 84.75166035493216, + "kd/rkl_advantage_std": 33.47325018843015, + "kd/ssd_loss": 0.0, + "learning_rate": 4.463600805073482e-07, + "loss": -14.772159830729167, + "num_tokens": 176361217.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.014656197528044384, + "sampling/importance_sampling_ratio/max": 1.027747905254364, + "sampling/importance_sampling_ratio/mean": 0.9998179952303569, + "sampling/importance_sampling_ratio/min": 0.9405593077341715, + "sampling/sampling_logp_difference/max": 0.07652996244529883, + "sampling/sampling_logp_difference/mean": 0.0012862812276580372, + "step": 14580, + "step_time": 4.414937950014913, + "student/entropy": 0.04743657391518354 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.266666666666667, + "completions/max_terminated_length": 5.266666666666667, + "completions/mean_length": 1.052222243944804, + "completions/mean_terminated_length": 1.052222243944804, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04054235400011142, + "epoch": 0.5548171495841719, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.783382415771484, + "kd/policy_loss": -0.7753073732058208, + "kd/rkl_advantage_mean": 36.66037802696228, + "kd/rkl_advantage_p95": 50.76131047407786, + "kd/rkl_advantage_std": 23.148512456317743, + "kd/ssd_loss": 0.0, + "learning_rate": 4.4522082558007065e-07, + "loss": -3.1012283325195313, + "num_tokens": 176693949.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0317885518074035, + "sampling/importance_sampling_ratio/mean": 0.9999136646588643, + "sampling/importance_sampling_ratio/min": 0.9604705254236857, + "sampling/sampling_logp_difference/max": 0.06311558762875696, + "sampling/sampling_logp_difference/mean": 0.0008629732479069692, + "step": 14610, + "step_time": 4.647906075255014, + "student/entropy": 0.04054235400011142 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.033333333333333, + "completions/max_terminated_length": 7.033333333333333, + "completions/mean_length": 1.0769444704055786, + "completions/mean_terminated_length": 1.0769444704055786, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05304263839498162, + "epoch": 0.5559564045114496, + "eval/gt_acc_batch": 0.0019444445768992105, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.19255828857422, + "kd/policy_loss": -3.488406030337016, + "kd/rkl_advantage_mean": 56.697931337356565, + "kd/rkl_advantage_p95": 73.40007422765096, + "kd/rkl_advantage_std": 27.852809714277587, + "kd/ssd_loss": 0.0, + "learning_rate": 4.4408157065279307e-07, + "loss": -13.953624471028645, + "num_tokens": 177002554.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445148110389, + "rewards/gt_logging_reward/std": 0.017397598922252656, + "sampling/importance_sampling_ratio/max": 1.0400423725446066, + "sampling/importance_sampling_ratio/mean": 1.000124837954839, + "sampling/importance_sampling_ratio/min": 0.9454384903113048, + "sampling/sampling_logp_difference/max": 0.0861432098162671, + "sampling/sampling_logp_difference/mean": 0.001683128131844569, + "step": 14640, + "step_time": 4.52579115460782, + "student/entropy": 0.05304263839498162 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.1, + "completions/max_terminated_length": 7.1, + "completions/mean_length": 1.0908333659172058, + "completions/mean_terminated_length": 1.0908333659172058, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0635764808083574, + "epoch": 0.5570956594387271, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.336551666259766, + "kd/policy_loss": -3.7106534242630005, + "kd/rkl_advantage_mean": 57.31156857808431, + "kd/rkl_advantage_p95": 73.21525608698526, + "kd/rkl_advantage_std": 27.246061034997304, + "kd/ssd_loss": 0.0, + "learning_rate": 4.429423157255155e-07, + "loss": -14.842616780598958, + "num_tokens": 177293889.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0383277773857116, + "sampling/importance_sampling_ratio/mean": 1.0001227299372355, + "sampling/importance_sampling_ratio/min": 0.9543647964795431, + "sampling/sampling_logp_difference/max": 0.07595669873990119, + "sampling/sampling_logp_difference/mean": 0.0016029632757029806, + "step": 14670, + "step_time": 4.3847374570245545, + "student/entropy": 0.0635764808083574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.166666666666667, + "completions/max_terminated_length": 6.166666666666667, + "completions/mean_length": 1.0486111323038736, + "completions/mean_terminated_length": 1.0486111323038736, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06342259161174298, + "epoch": 0.5582349143660046, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 29.93808937072754, + "kd/policy_loss": -1.2412327885627747, + "kd/rkl_advantage_mean": 49.151962677637734, + "kd/rkl_advantage_p95": 67.54195973873138, + "kd/rkl_advantage_std": 30.454077504078548, + "kd/ssd_loss": 0.0, + "learning_rate": 4.418030607982379e-07, + "loss": -4.964932250976562, + "num_tokens": 177608488.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0216776251792907, + "sampling/importance_sampling_ratio/mean": 1.0001201411088307, + "sampling/importance_sampling_ratio/min": 0.9646915415922801, + "sampling/sampling_logp_difference/max": 0.06084546878313025, + "sampling/sampling_logp_difference/mean": 0.0010876918958577638, + "step": 14700, + "step_time": 4.54692629877245, + "student/entropy": 0.06342259161174298 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.633333333333334, + "completions/max_terminated_length": 7.633333333333334, + "completions/mean_length": 1.113888931274414, + "completions/mean_terminated_length": 1.113888931274414, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.065225533892711, + "epoch": 0.5593741692932822, + "eval/gt_acc_batch": 0.0025000001303851606, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.986725807189941, + "kd/policy_loss": -6.639724898338318, + "kd/rkl_advantage_mean": 69.61405021349589, + "kd/rkl_advantage_p95": 84.61089136600495, + "kd/rkl_advantage_std": 26.844129846493402, + "kd/ssd_loss": 0.0, + "learning_rate": 4.4066380587096037e-07, + "loss": -26.558902994791666, + "num_tokens": 177911450.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000001303851606, + "rewards/gt_logging_reward/std": 0.025585518529017765, + "sampling/importance_sampling_ratio/max": 1.045940891901652, + "sampling/importance_sampling_ratio/mean": 0.9998556991418203, + "sampling/importance_sampling_ratio/min": 0.9324764509995779, + "sampling/sampling_logp_difference/max": 0.08101071060324709, + "sampling/sampling_logp_difference/mean": 0.0016264540482855713, + "step": 14730, + "step_time": 4.578479123057332, + "student/entropy": 0.065225533892711 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.0577778021494548, + "completions/mean_terminated_length": 1.0577778021494548, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.055222318942348164, + "epoch": 0.5605134242205597, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.906614303588867, + "kd/policy_loss": -1.3490427474180857, + "kd/rkl_advantage_mean": 48.41048995653788, + "kd/rkl_advantage_p95": 67.30741961797078, + "kd/rkl_advantage_std": 29.60467787384987, + "kd/ssd_loss": 0.0, + "learning_rate": 4.395245509436828e-07, + "loss": -5.396172078450521, + "num_tokens": 178218650.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0406634569168092, + "sampling/importance_sampling_ratio/mean": 1.0002752562363944, + "sampling/importance_sampling_ratio/min": 0.9672892530759175, + "sampling/sampling_logp_difference/max": 0.06193421874195337, + "sampling/sampling_logp_difference/mean": 0.0011529385543932829, + "step": 14760, + "step_time": 4.447771670890506, + "student/entropy": 0.055222318942348164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.3, + "completions/max_terminated_length": 8.3, + "completions/mean_length": 1.0947222550710043, + "completions/mean_terminated_length": 1.0947222550710043, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04634717317918936, + "epoch": 0.5616526791478373, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.473602294921875, + "kd/policy_loss": -4.319095245997111, + "kd/rkl_advantage_mean": 70.8297289689382, + "kd/rkl_advantage_p95": 90.20543892383576, + "kd/rkl_advantage_std": 34.35777384340763, + "kd/ssd_loss": 0.0, + "learning_rate": 4.383852960164052e-07, + "loss": -17.27637939453125, + "num_tokens": 178525975.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.034498405456543, + "sampling/importance_sampling_ratio/mean": 0.9997164487838746, + "sampling/importance_sampling_ratio/min": 0.9381605724493662, + "sampling/sampling_logp_difference/max": 0.07541204650575915, + "sampling/sampling_logp_difference/mean": 0.001429075719594645, + "step": 14790, + "step_time": 4.529581057664473, + "student/entropy": 0.04634717317918936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.766666666666667, + "completions/max_terminated_length": 6.766666666666667, + "completions/mean_length": 1.063611129919688, + "completions/mean_terminated_length": 1.063611129919688, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04070007422318061, + "epoch": 0.5627919340751149, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.388090133666992, + "kd/policy_loss": -1.8368135650952657, + "kd/rkl_advantage_mean": 50.96275544166565, + "kd/rkl_advantage_p95": 67.50590405464172, + "kd/rkl_advantage_std": 28.527141270041465, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3724604108912773e-07, + "loss": -7.34725341796875, + "num_tokens": 178834620.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0758762240409852, + "sampling/importance_sampling_ratio/mean": 1.000483125448227, + "sampling/importance_sampling_ratio/min": 0.977240389585495, + "sampling/sampling_logp_difference/max": 0.07491945582441986, + "sampling/sampling_logp_difference/mean": 0.000990060622279998, + "step": 14820, + "step_time": 4.428456004126929, + "student/entropy": 0.04070007422318061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.833333333333333, + "completions/max_terminated_length": 7.833333333333333, + "completions/mean_length": 1.0816667040189107, + "completions/mean_terminated_length": 1.0816667040189107, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04547802129139503, + "epoch": 0.5639311890023925, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.759271621704102, + "kd/policy_loss": -4.389346772432328, + "kd/rkl_advantage_mean": 73.22499114672343, + "kd/rkl_advantage_p95": 95.83531999588013, + "kd/rkl_advantage_std": 37.95203857322534, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3610678616185015e-07, + "loss": -17.55738728841146, + "num_tokens": 179149698.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0745391885439555, + "sampling/importance_sampling_ratio/mean": 1.0001939594745637, + "sampling/importance_sampling_ratio/min": 0.9486244440078735, + "sampling/sampling_logp_difference/max": 0.09960451334094007, + "sampling/sampling_logp_difference/mean": 0.0014827509449484447, + "step": 14850, + "step_time": 4.610220651200507, + "student/entropy": 0.04547802129139503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.766666666666667, + "completions/max_terminated_length": 4.766666666666667, + "completions/mean_length": 1.0313888986905415, + "completions/mean_terminated_length": 1.0313888986905415, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.038470133425047, + "epoch": 0.56507044392967, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.84667205810547, + "kd/policy_loss": 1.3458776036898294, + "kd/rkl_advantage_mean": 25.642144759496052, + "kd/rkl_advantage_p95": 39.471954981486, + "kd/rkl_advantage_std": 21.49139608393113, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3496753123457257e-07, + "loss": 5.383512369791666, + "num_tokens": 179452723.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0181734442710877, + "sampling/importance_sampling_ratio/mean": 1.0000821312268575, + "sampling/importance_sampling_ratio/min": 0.9837234040101369, + "sampling/sampling_logp_difference/max": 0.03220798223434637, + "sampling/sampling_logp_difference/mean": 0.0005914934673152554, + "step": 14880, + "step_time": 4.53067599494243, + "student/entropy": 0.038470133425047 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.066666666666666, + "completions/max_terminated_length": 6.066666666666666, + "completions/mean_length": 1.0708333571751913, + "completions/mean_terminated_length": 1.0708333571751913, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06495284931734205, + "epoch": 0.5662096988569476, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 27.858749389648438, + "kd/policy_loss": -2.415462402502696, + "kd/rkl_advantage_mean": 47.36476157506307, + "kd/rkl_advantage_p95": 62.26559438705444, + "kd/rkl_advantage_std": 24.857211503386498, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3382827630729504e-07, + "loss": -9.661849975585938, + "num_tokens": 179753498.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0164183497428894, + "sampling/importance_sampling_ratio/mean": 0.9998338361581166, + "sampling/importance_sampling_ratio/min": 0.9439894179503123, + "sampling/sampling_logp_difference/max": 0.06359924289087454, + "sampling/sampling_logp_difference/mean": 0.0012824941591437285, + "step": 14910, + "step_time": 4.506340073111157, + "student/entropy": 0.06495284931734205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.7, + "completions/max_terminated_length": 8.7, + "completions/mean_length": 1.1119444767634075, + "completions/mean_terminated_length": 1.1119444767634075, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.060419538989663124, + "epoch": 0.5673489537842251, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.638899803161621, + "kd/policy_loss": -4.853962604204813, + "kd/rkl_advantage_mean": 76.03020896911622, + "kd/rkl_advantage_p95": 96.28895308176676, + "kd/rkl_advantage_std": 35.90400481025378, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3268902138001745e-07, + "loss": -19.415848795572916, + "num_tokens": 180067701.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0297939618428549, + "sampling/importance_sampling_ratio/mean": 0.9997384230295817, + "sampling/importance_sampling_ratio/min": 0.9223570168018341, + "sampling/sampling_logp_difference/max": 0.09995873172301799, + "sampling/sampling_logp_difference/mean": 0.0016643805720377713, + "step": 14940, + "step_time": 4.490343916396766, + "student/entropy": 0.060419538989663124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.733333333333333, + "completions/max_terminated_length": 7.733333333333333, + "completions/mean_length": 1.0747222503026326, + "completions/mean_terminated_length": 1.0747222503026326, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048993152876695, + "epoch": 0.5684882087115026, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.072229385375977, + "kd/policy_loss": -2.3598785599072776, + "kd/rkl_advantage_mean": 56.37349088986715, + "kd/rkl_advantage_p95": 73.43521352608998, + "kd/rkl_advantage_std": 30.231410696109137, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3154976645273987e-07, + "loss": -9.439515177408854, + "num_tokens": 180372074.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0278599977493286, + "sampling/importance_sampling_ratio/mean": 1.0000984966754913, + "sampling/importance_sampling_ratio/min": 0.9596699774265289, + "sampling/sampling_logp_difference/max": 0.07372733977002402, + "sampling/sampling_logp_difference/mean": 0.0011174705189963181, + "step": 14970, + "step_time": 4.618676488062677, + "student/entropy": 0.048993152876695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.033333333333333, + "completions/max_terminated_length": 7.033333333333333, + "completions/mean_length": 1.0586111346880596, + "completions/mean_terminated_length": 1.0586111346880596, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04995404494305452, + "epoch": 0.5696274636387803, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 41.156044006347656, + "kd/policy_loss": -1.6392152945200602, + "kd/rkl_advantage_mean": 55.01425059636434, + "kd/rkl_advantage_p95": 73.7860128323237, + "kd/rkl_advantage_std": 32.050319221615794, + "kd/ssd_loss": 0.0, + "learning_rate": 4.304105115254623e-07, + "loss": -6.5568603515625, + "num_tokens": 180685293.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0291231711705526, + "sampling/importance_sampling_ratio/mean": 0.9999595204989116, + "sampling/importance_sampling_ratio/min": 0.9477235158284505, + "sampling/sampling_logp_difference/max": 0.0891359698648254, + "sampling/sampling_logp_difference/mean": 0.0012742472220755492, + "step": 15000, + "step_time": 4.557632566181322, + "student/entropy": 0.04995404494305452 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.833333333333333, + "completions/max_terminated_length": 6.833333333333333, + "completions/mean_length": 1.0627778013547262, + "completions/mean_terminated_length": 1.0627778013547262, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04863918038705985, + "epoch": 0.5707667185660578, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 26.51312255859375, + "kd/policy_loss": -2.270056398709615, + "kd/rkl_advantage_mean": 55.18519654273987, + "kd/rkl_advantage_p95": 73.23790238698324, + "kd/rkl_advantage_std": 30.511188120643297, + "kd/ssd_loss": 0.0, + "learning_rate": 4.2927125659818476e-07, + "loss": -9.080226643880208, + "num_tokens": 180994135.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0286218484242757, + "sampling/importance_sampling_ratio/mean": 0.9998931407928466, + "sampling/importance_sampling_ratio/min": 0.9360958576202393, + "sampling/sampling_logp_difference/max": 0.06793836033903063, + "sampling/sampling_logp_difference/mean": 0.0012712143293659513, + "step": 15030, + "step_time": 4.632772342301905, + "student/entropy": 0.04863918038705985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.933333333333334, + "completions/max_terminated_length": 5.933333333333334, + "completions/mean_length": 1.0475000222524007, + "completions/mean_terminated_length": 1.0475000222524007, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05514357623954614, + "epoch": 0.5719059734933354, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 24.400146484375, + "kd/policy_loss": -1.2309174855550131, + "kd/rkl_advantage_mean": 45.45850830078125, + "kd/rkl_advantage_p95": 62.22490693728129, + "kd/rkl_advantage_std": 27.87392311791579, + "kd/ssd_loss": 0.0, + "learning_rate": 4.281320016709072e-07, + "loss": -4.923670959472656, + "num_tokens": 181302194.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0053771615028382, + "sampling/importance_sampling_ratio/mean": 0.9998238841692607, + "sampling/importance_sampling_ratio/min": 0.9490409195423126, + "sampling/sampling_logp_difference/max": 0.05544694853791346, + "sampling/sampling_logp_difference/mean": 0.0010647053093028565, + "step": 15060, + "step_time": 4.458954004892925, + "student/entropy": 0.05514357623954614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.833333333333333, + "completions/max_terminated_length": 7.833333333333333, + "completions/mean_length": 1.0838889201482138, + "completions/mean_terminated_length": 1.0838889201482138, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05437894190351168, + "epoch": 0.5730452284206129, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 43.44954299926758, + "kd/policy_loss": -4.638324318329493, + "kd/rkl_advantage_mean": 73.21421791712443, + "kd/rkl_advantage_p95": 95.91195289293925, + "kd/rkl_advantage_std": 37.23059570689996, + "kd/ssd_loss": 0.0, + "learning_rate": 4.2699274674362965e-07, + "loss": -18.5532958984375, + "num_tokens": 181611848.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0420330484708151, + "sampling/importance_sampling_ratio/mean": 0.9999269604682922, + "sampling/importance_sampling_ratio/min": 0.9324882348378499, + "sampling/sampling_logp_difference/max": 0.0918337996583432, + "sampling/sampling_logp_difference/mean": 0.001415857042108352, + "step": 15090, + "step_time": 4.495598126074765, + "student/entropy": 0.05437894190351168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.433333333333334, + "completions/max_terminated_length": 6.433333333333334, + "completions/mean_length": 1.0611111362775167, + "completions/mean_terminated_length": 1.0611111362775167, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04104980166691045, + "epoch": 0.5741844833478905, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.24706745147705, + "kd/policy_loss": -1.5509857257207236, + "kd/rkl_advantage_mean": 49.824467261632286, + "kd/rkl_advantage_p95": 67.7259476184845, + "kd/rkl_advantage_std": 29.59271642367045, + "kd/ssd_loss": 0.0, + "learning_rate": 4.258534918163521e-07, + "loss": -6.203940836588542, + "num_tokens": 181914548.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0362293720245361, + "sampling/importance_sampling_ratio/mean": 1.000227504968643, + "sampling/importance_sampling_ratio/min": 0.9726442396640778, + "sampling/sampling_logp_difference/max": 0.05346369938924909, + "sampling/sampling_logp_difference/mean": 0.0008371788988976429, + "step": 15120, + "step_time": 4.424656167424595, + "student/entropy": 0.04104980166691045 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.7, + "completions/max_terminated_length": 5.7, + "completions/mean_length": 1.0441666841506958, + "completions/mean_terminated_length": 1.0441666841506958, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04799175470446547, + "epoch": 0.575323738275168, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.5159330368042, + "kd/policy_loss": -0.4675127506256104, + "kd/rkl_advantage_mean": 40.123833656311035, + "kd/rkl_advantage_p95": 56.13238895734151, + "kd/rkl_advantage_std": 26.273824245731035, + "kd/ssd_loss": 0.0, + "learning_rate": 4.2471423688907454e-07, + "loss": -1.8700520833333334, + "num_tokens": 182222707.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.026047662893931, + "sampling/importance_sampling_ratio/mean": 1.0001297771930695, + "sampling/importance_sampling_ratio/min": 0.9713639756043752, + "sampling/sampling_logp_difference/max": 0.05168696095158035, + "sampling/sampling_logp_difference/mean": 0.000839926317469993, + "step": 15150, + "step_time": 4.526620359641189, + "student/entropy": 0.04799175470446547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.166666666666667, + "completions/max_terminated_length": 7.166666666666667, + "completions/mean_length": 1.0725000341733297, + "completions/mean_terminated_length": 1.0725000341733297, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.060130851436406375, + "epoch": 0.5764629932024457, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.058626174926758, + "kd/policy_loss": -3.33316179116567, + "kd/rkl_advantage_mean": 63.90335620244344, + "kd/rkl_advantage_p95": 84.64012573560079, + "kd/rkl_advantage_std": 34.572999983032545, + "kd/ssd_loss": 0.0, + "learning_rate": 4.2357498196179695e-07, + "loss": -13.33264872233073, + "num_tokens": 182532136.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0354691743850708, + "sampling/importance_sampling_ratio/mean": 1.0002303898334504, + "sampling/importance_sampling_ratio/min": 0.9734186271826426, + "sampling/sampling_logp_difference/max": 0.057624451055501895, + "sampling/sampling_logp_difference/mean": 0.001124387669066588, + "step": 15180, + "step_time": 4.429489852949822, + "student/entropy": 0.060130851436406375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.366666666666666, + "completions/max_terminated_length": 7.366666666666666, + "completions/mean_length": 1.087777805328369, + "completions/mean_terminated_length": 1.087777805328369, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054813009190062684, + "epoch": 0.5776022481297232, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.65433120727539, + "kd/policy_loss": -3.6040438175201417, + "kd/rkl_advantage_mean": 62.01097612380981, + "kd/rkl_advantage_p95": 79.28878109455108, + "kd/rkl_advantage_std": 30.802145685752233, + "kd/ssd_loss": 0.0, + "learning_rate": 4.224357270345194e-07, + "loss": -14.416176350911458, + "num_tokens": 182840204.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0782318353652953, + "sampling/importance_sampling_ratio/mean": 1.0003749132156372, + "sampling/importance_sampling_ratio/min": 0.9466482559839885, + "sampling/sampling_logp_difference/max": 0.10300817292494079, + "sampling/sampling_logp_difference/mean": 0.0016500333974060293, + "step": 15210, + "step_time": 4.495108500115263, + "student/entropy": 0.054813009190062684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.133333333333334, + "completions/max_terminated_length": 7.133333333333334, + "completions/mean_length": 1.0655555804570516, + "completions/mean_terminated_length": 1.0655555804570516, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04822232403482, + "epoch": 0.5787415030570007, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.048787117004395, + "kd/policy_loss": -2.5877371629079184, + "kd/rkl_advantage_mean": 59.51529572804769, + "kd/rkl_advantage_p95": 78.96405079364777, + "kd/rkl_advantage_std": 32.91794673800469, + "kd/ssd_loss": 0.0, + "learning_rate": 4.2129647210724184e-07, + "loss": -10.350948079427083, + "num_tokens": 183153016.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0244015773137412, + "sampling/importance_sampling_ratio/mean": 0.9999405264854431, + "sampling/importance_sampling_ratio/min": 0.9500062425931295, + "sampling/sampling_logp_difference/max": 0.07515022975082199, + "sampling/sampling_logp_difference/mean": 0.0011609735006156067, + "step": 15240, + "step_time": 4.507499105696722, + "student/entropy": 0.04822232403482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0622222463289896, + "completions/mean_terminated_length": 1.0622222463289896, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04270975810165207, + "epoch": 0.5798807579842783, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.380374908447266, + "kd/policy_loss": -1.893544872601827, + "kd/rkl_advantage_mean": 50.490130790074666, + "kd/rkl_advantage_p95": 67.73835634390512, + "kd/rkl_advantage_std": 28.784240397810937, + "kd/ssd_loss": 0.0, + "learning_rate": 4.2015721717996426e-07, + "loss": -7.574180094401042, + "num_tokens": 183451128.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0111658175786336, + "sampling/importance_sampling_ratio/mean": 0.999701976776123, + "sampling/importance_sampling_ratio/min": 0.941748970746994, + "sampling/sampling_logp_difference/max": 0.06493961185527344, + "sampling/sampling_logp_difference/mean": 0.0009492243989370763, + "step": 15270, + "step_time": 4.541138478907912, + "student/entropy": 0.04270975810165207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.055833355585734, + "completions/mean_terminated_length": 1.055833355585734, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04793878852700194, + "epoch": 0.5810200129115558, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.33798599243164, + "kd/policy_loss": -1.3656395117441813, + "kd/rkl_advantage_mean": 54.029023504257204, + "kd/rkl_advantage_p95": 73.64608959356944, + "kd/rkl_advantage_std": 33.012674730022745, + "kd/ssd_loss": 0.0, + "learning_rate": 4.1901796225268673e-07, + "loss": -5.46255849202474, + "num_tokens": 183762481.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0371957540512085, + "sampling/importance_sampling_ratio/mean": 1.00023699204127, + "sampling/importance_sampling_ratio/min": 0.9656885862350464, + "sampling/sampling_logp_difference/max": 0.06546834511682391, + "sampling/sampling_logp_difference/mean": 0.0010238474341652667, + "step": 15300, + "step_time": 4.686368879966904, + "student/entropy": 0.04793878852700194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0722222526868184, + "completions/mean_terminated_length": 1.0722222526868184, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054696235619485375, + "epoch": 0.5821592678388334, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 39.05929946899414, + "kd/policy_loss": -2.718185615539551, + "kd/rkl_advantage_mean": 59.57203729947408, + "kd/rkl_advantage_p95": 78.83294754823049, + "kd/rkl_advantage_std": 32.95139507353306, + "kd/ssd_loss": 0.0, + "learning_rate": 4.1787870732540915e-07, + "loss": -10.872743733723958, + "num_tokens": 184064029.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.02907608350118, + "sampling/importance_sampling_ratio/mean": 0.9999552587668101, + "sampling/importance_sampling_ratio/min": 0.9399492541948954, + "sampling/sampling_logp_difference/max": 0.0883939029959341, + "sampling/sampling_logp_difference/mean": 0.0014229108754079788, + "step": 15330, + "step_time": 4.676657231675927, + "student/entropy": 0.054696235619485375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.233333333333333, + "completions/max_terminated_length": 6.233333333333333, + "completions/mean_length": 1.0766667008399964, + "completions/mean_terminated_length": 1.0766667008399964, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05681987457598249, + "epoch": 0.5832985227661109, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 31.501474380493164, + "kd/policy_loss": -2.784126849969228, + "kd/rkl_advantage_mean": 51.11089436213175, + "kd/rkl_advantage_p95": 67.93056461016337, + "kd/rkl_advantage_std": 27.3828387260437, + "kd/ssd_loss": 0.0, + "learning_rate": 4.167394523981316e-07, + "loss": -11.136507161458333, + "num_tokens": 184370665.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0172185142834982, + "sampling/importance_sampling_ratio/mean": 0.9998211046059926, + "sampling/importance_sampling_ratio/min": 0.9433641453584035, + "sampling/sampling_logp_difference/max": 0.07307508536614478, + "sampling/sampling_logp_difference/mean": 0.0013259448790146659, + "step": 15360, + "step_time": 4.680823073198553, + "student/entropy": 0.05681987457598249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.4, + "completions/max_terminated_length": 7.4, + "completions/mean_length": 1.0619444727897644, + "completions/mean_terminated_length": 1.0619444727897644, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052891679542760056, + "epoch": 0.5844377776933886, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.233123779296875, + "kd/policy_loss": -2.3795761783917744, + "kd/rkl_advantage_mean": 65.69812839825948, + "kd/rkl_advantage_p95": 90.63792202472686, + "kd/rkl_advantage_std": 39.64680834611257, + "kd/ssd_loss": 0.0, + "learning_rate": 4.1560019747085404e-07, + "loss": -9.518304443359375, + "num_tokens": 184688856.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0097639679908752, + "sampling/importance_sampling_ratio/mean": 0.9998646179835001, + "sampling/importance_sampling_ratio/min": 0.9490215281645457, + "sampling/sampling_logp_difference/max": 0.07321543407936891, + "sampling/sampling_logp_difference/mean": 0.0011701839733480786, + "step": 15390, + "step_time": 4.949043329758569, + "student/entropy": 0.052891679542760056 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0677778045336406, + "completions/mean_terminated_length": 1.0677778045336406, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04029178769948582, + "epoch": 0.5855770326206661, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.762874603271484, + "kd/policy_loss": -2.326912327607473, + "kd/rkl_advantage_mean": 58.88903415997823, + "kd/rkl_advantage_p95": 78.92207384109497, + "kd/rkl_advantage_std": 33.84470949073633, + "kd/ssd_loss": 0.0, + "learning_rate": 4.144609425435765e-07, + "loss": -9.307651774088542, + "num_tokens": 184999548.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0104031403859457, + "sampling/importance_sampling_ratio/mean": 0.999827367067337, + "sampling/importance_sampling_ratio/min": 0.9575987299283345, + "sampling/sampling_logp_difference/max": 0.05488903296645731, + "sampling/sampling_logp_difference/mean": 0.0009540133803966455, + "step": 15420, + "step_time": 4.7589055652225705, + "student/entropy": 0.04029178769948582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.7, + "completions/max_terminated_length": 6.7, + "completions/mean_length": 1.0644444704055787, + "completions/mean_terminated_length": 1.0644444704055787, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04687512905026476, + "epoch": 0.5867162875479437, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.966611862182617, + "kd/policy_loss": -2.2543189843495686, + "kd/rkl_advantage_mean": 51.605441617965695, + "kd/rkl_advantage_p95": 68.07803622086843, + "kd/rkl_advantage_std": 27.991451113422713, + "kd/ssd_loss": 0.0, + "learning_rate": 4.133216876162989e-07, + "loss": -9.017277018229167, + "num_tokens": 185301836.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0311944921811422, + "sampling/importance_sampling_ratio/mean": 1.00024218360583, + "sampling/importance_sampling_ratio/min": 0.9768406192461649, + "sampling/sampling_logp_difference/max": 0.059256594209000465, + "sampling/sampling_logp_difference/mean": 0.0009455011585184063, + "step": 15450, + "step_time": 4.666978817374911, + "student/entropy": 0.04687512905026476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.2, + "completions/max_terminated_length": 5.2, + "completions/mean_length": 1.0588889082272848, + "completions/mean_terminated_length": 1.0588889082272848, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054537130737056336, + "epoch": 0.5878555424752212, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 67.70970916748047, + "kd/policy_loss": -1.6561713854471842, + "kd/rkl_advantage_mean": 38.28552393913269, + "kd/rkl_advantage_p95": 50.68585631052653, + "kd/rkl_advantage_std": 21.140471765398978, + "kd/ssd_loss": 0.0, + "learning_rate": 4.1218243268902134e-07, + "loss": -6.624685668945313, + "num_tokens": 185607072.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.023829185962677, + "sampling/importance_sampling_ratio/mean": 1.000137980779012, + "sampling/importance_sampling_ratio/min": 0.9710213939348856, + "sampling/sampling_logp_difference/max": 0.048488814197480676, + "sampling/sampling_logp_difference/mean": 0.0009067395837822308, + "step": 15480, + "step_time": 4.39909727225701, + "student/entropy": 0.054537130737056336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.533333333333333, + "completions/max_terminated_length": 7.533333333333333, + "completions/mean_length": 1.0650000294049582, + "completions/mean_terminated_length": 1.0650000294049582, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05849583645661672, + "epoch": 0.5889947974024987, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.366495132446289, + "kd/policy_loss": -2.4374633908271788, + "kd/rkl_advantage_mean": 63.801388057072955, + "kd/rkl_advantage_p95": 84.81865318616231, + "kd/rkl_advantage_std": 36.28714383840561, + "kd/ssd_loss": 0.0, + "learning_rate": 4.110431777617438e-07, + "loss": -9.749853515625, + "num_tokens": 185922538.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.024151118596395, + "sampling/importance_sampling_ratio/mean": 0.9997852663199107, + "sampling/importance_sampling_ratio/min": 0.9331821004549662, + "sampling/sampling_logp_difference/max": 0.06887392071463788, + "sampling/sampling_logp_difference/mean": 0.0011791485148326805, + "step": 15510, + "step_time": 4.70458315051316, + "student/entropy": 0.05849583645661672 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.7, + "completions/max_terminated_length": 6.7, + "completions/mean_length": 1.067222249507904, + "completions/mean_terminated_length": 1.067222249507904, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049711887693653506, + "epoch": 0.5901340523297763, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.15995979309082, + "kd/policy_loss": -1.8674968322118124, + "kd/rkl_advantage_mean": 54.42006163597107, + "kd/rkl_advantage_p95": 72.82284779548645, + "kd/rkl_advantage_std": 31.655413577953976, + "kd/ssd_loss": 0.0, + "learning_rate": 4.0990392283446623e-07, + "loss": -7.469990030924479, + "num_tokens": 186232756.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0397975842158, + "sampling/importance_sampling_ratio/mean": 0.9999944567680359, + "sampling/importance_sampling_ratio/min": 0.9369258105754852, + "sampling/sampling_logp_difference/max": 0.07831576395158966, + "sampling/sampling_logp_difference/mean": 0.0013252822012873367, + "step": 15540, + "step_time": 4.563036664831452, + "student/entropy": 0.049711887693653506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.366666666666666, + "completions/max_terminated_length": 7.366666666666666, + "completions/mean_length": 1.0797222574551901, + "completions/mean_terminated_length": 1.0797222574551901, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05889642769470811, + "epoch": 0.5912733072570538, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 142.39430236816406, + "kd/policy_loss": -2.7230833848317464, + "kd/rkl_advantage_mean": 63.363940699895224, + "kd/rkl_advantage_p95": 84.59861691792806, + "kd/rkl_advantage_std": 35.874806456267834, + "kd/ssd_loss": 0.0, + "learning_rate": 4.0876466790718865e-07, + "loss": -10.892337036132812, + "num_tokens": 186539819.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0711971600850423, + "sampling/importance_sampling_ratio/mean": 1.0002456168333689, + "sampling/importance_sampling_ratio/min": 0.9440463860829671, + "sampling/sampling_logp_difference/max": 0.09488713890314102, + "sampling/sampling_logp_difference/mean": 0.0016072729951702059, + "step": 15570, + "step_time": 4.73975230330058, + "student/entropy": 0.05889642769470811 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.666666666666667, + "completions/max_terminated_length": 6.666666666666667, + "completions/mean_length": 1.0777778029441833, + "completions/mean_terminated_length": 1.0777778029441833, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06540845396618049, + "epoch": 0.5924125621843315, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 173.3645782470703, + "kd/policy_loss": -2.246100322405497, + "kd/rkl_advantage_mean": 51.42740144729614, + "kd/rkl_advantage_p95": 67.73563674290975, + "kd/rkl_advantage_std": 27.86087395846844, + "kd/ssd_loss": 0.0, + "learning_rate": 4.076254129799111e-07, + "loss": -8.984403483072917, + "num_tokens": 186848331.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0377585967381795, + "sampling/importance_sampling_ratio/mean": 0.9999563256899516, + "sampling/importance_sampling_ratio/min": 0.9406979819138844, + "sampling/sampling_logp_difference/max": 0.08653856629195313, + "sampling/sampling_logp_difference/mean": 0.0014066703801897044, + "step": 15600, + "step_time": 4.6010491467701895, + "student/entropy": 0.06540845396618049 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.833333333333333, + "completions/max_terminated_length": 7.833333333333333, + "completions/mean_length": 1.083055591583252, + "completions/mean_terminated_length": 1.083055591583252, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.058320761161545914, + "epoch": 0.593551817111609, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.751498222351074, + "kd/policy_loss": -4.061229070027669, + "kd/rkl_advantage_mean": 70.15439939498901, + "kd/rkl_advantage_p95": 90.45126167138417, + "kd/rkl_advantage_std": 35.675173324346545, + "kd/ssd_loss": 0.0, + "learning_rate": 4.0648615805263354e-07, + "loss": -16.24491678873698, + "num_tokens": 187163846.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0731238683064779, + "sampling/importance_sampling_ratio/mean": 1.0003287057081858, + "sampling/importance_sampling_ratio/min": 0.9621389746665955, + "sampling/sampling_logp_difference/max": 0.08832120193789403, + "sampling/sampling_logp_difference/mean": 0.0012656635750317945, + "step": 15630, + "step_time": 4.770085640542675, + "student/entropy": 0.058320761161545914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.733333333333333, + "completions/max_terminated_length": 7.733333333333333, + "completions/mean_length": 1.0838889241218568, + "completions/mean_terminated_length": 1.0838889241218568, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.051802266668528316, + "epoch": 0.5946910720388866, + "eval/gt_acc_batch": 0.0025000001303851606, + "frac_reward_zero_std": 1.0, + "grad_norm": 42.610801696777344, + "kd/policy_loss": -3.279638091723124, + "kd/rkl_advantage_mean": 67.76933107376098, + "kd/rkl_advantage_p95": 90.48494311173756, + "kd/rkl_advantage_std": 37.46641492346922, + "kd/ssd_loss": 0.0, + "learning_rate": 4.05346903125356e-07, + "loss": -13.118553670247396, + "num_tokens": 187468732.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000001303851606, + "rewards/gt_logging_reward/std": 0.02378490741054217, + "sampling/importance_sampling_ratio/max": 1.061638883749644, + "sampling/importance_sampling_ratio/mean": 1.0002688407897948, + "sampling/importance_sampling_ratio/min": 0.9586070040861766, + "sampling/sampling_logp_difference/max": 0.08657768491345147, + "sampling/sampling_logp_difference/mean": 0.0015084233125283693, + "step": 15660, + "step_time": 4.734845536896803, + "student/entropy": 0.051802266668528316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.466666666666667, + "completions/max_terminated_length": 4.466666666666667, + "completions/mean_length": 1.0469444632530212, + "completions/mean_terminated_length": 1.0469444632530212, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0470720042164127, + "epoch": 0.5958303269661641, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.037291526794434, + "kd/policy_loss": 0.048618217309316, + "kd/rkl_advantage_mean": 27.49215251604716, + "kd/rkl_advantage_p95": 39.51732378005981, + "kd/rkl_advantage_std": 19.35422018667062, + "kd/ssd_loss": 0.0, + "learning_rate": 4.042076481980785e-07, + "loss": 0.19447344144185383, + "num_tokens": 187777605.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0083194653193155, + "sampling/importance_sampling_ratio/mean": 0.9995431741078694, + "sampling/importance_sampling_ratio/min": 0.9335480709870656, + "sampling/sampling_logp_difference/max": 0.07188753414278229, + "sampling/sampling_logp_difference/mean": 0.001103141614779209, + "step": 15690, + "step_time": 4.75503718375694, + "student/entropy": 0.0470720042164127 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.4, + "completions/max_terminated_length": 7.4, + "completions/mean_length": 1.072777811686198, + "completions/mean_terminated_length": 1.072777811686198, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06458472618833185, + "epoch": 0.5969695818934417, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.063051223754883, + "kd/policy_loss": -3.6644495824972787, + "kd/rkl_advantage_mean": 67.50747810999552, + "kd/rkl_advantage_p95": 90.32028272151948, + "kd/rkl_advantage_std": 36.61966203351815, + "kd/ssd_loss": 0.0, + "learning_rate": 4.030683932708009e-07, + "loss": -14.657799275716146, + "num_tokens": 188082747.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0510780453681945, + "sampling/importance_sampling_ratio/mean": 0.9999150554339091, + "sampling/importance_sampling_ratio/min": 0.9397371371587118, + "sampling/sampling_logp_difference/max": 0.10064576086588203, + "sampling/sampling_logp_difference/mean": 0.0017595200624782592, + "step": 15720, + "step_time": 4.785884505459884, + "student/entropy": 0.06458472618833185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.133333333333333, + "completions/max_terminated_length": 8.133333333333333, + "completions/mean_length": 1.0783333659172059, + "completions/mean_terminated_length": 1.0783333659172059, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.055579230282455684, + "epoch": 0.5981088368207192, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.725167274475098, + "kd/policy_loss": -3.7093086083730062, + "kd/rkl_advantage_mean": 69.08855533599854, + "kd/rkl_advantage_p95": 90.27779920895894, + "kd/rkl_advantage_std": 36.0615927418073, + "kd/ssd_loss": 0.0, + "learning_rate": 4.019291383435233e-07, + "loss": -14.8372314453125, + "num_tokens": 188394805.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0215475718180338, + "sampling/importance_sampling_ratio/mean": 0.9997721413771311, + "sampling/importance_sampling_ratio/min": 0.9441535512606303, + "sampling/sampling_logp_difference/max": 0.06965457412103812, + "sampling/sampling_logp_difference/mean": 0.0010562885930994526, + "step": 15750, + "step_time": 4.788633482872198, + "student/entropy": 0.055579230282455684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.833333333333333, + "completions/max_terminated_length": 5.833333333333333, + "completions/mean_length": 1.0527778029441834, + "completions/mean_terminated_length": 1.0527778029441834, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04316496411338448, + "epoch": 0.5992480917479968, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.660659790039062, + "kd/policy_loss": -1.1792864203453064, + "kd/rkl_advantage_mean": 44.905358457565306, + "kd/rkl_advantage_p95": 61.81823639074961, + "kd/rkl_advantage_std": 27.91835122803847, + "kd/ssd_loss": 0.0, + "learning_rate": 4.0078988341624573e-07, + "loss": -4.717145792643229, + "num_tokens": 188697707.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0505269408226012, + "sampling/importance_sampling_ratio/mean": 1.0002076506614686, + "sampling/importance_sampling_ratio/min": 0.959653236468633, + "sampling/sampling_logp_difference/max": 0.07936240577449401, + "sampling/sampling_logp_difference/mean": 0.001105035439832136, + "step": 15780, + "step_time": 4.714930162373154, + "student/entropy": 0.04316496411338448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.0611111323038738, + "completions/mean_terminated_length": 1.0611111323038738, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04666216727346182, + "epoch": 0.6003873466752744, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.706833839416504, + "kd/policy_loss": -2.044444998105367, + "kd/rkl_advantage_mean": 59.49802268346151, + "kd/rkl_advantage_p95": 79.1272829135259, + "kd/rkl_advantage_std": 33.97206035157045, + "kd/ssd_loss": 0.0, + "learning_rate": 3.996506284889682e-07, + "loss": -8.177780151367188, + "num_tokens": 189009375.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0607900341351828, + "sampling/importance_sampling_ratio/mean": 1.0002887666225433, + "sampling/importance_sampling_ratio/min": 0.9551486293474833, + "sampling/sampling_logp_difference/max": 0.09224210353568196, + "sampling/sampling_logp_difference/mean": 0.0013717341954664639, + "step": 15810, + "step_time": 4.769947293963439, + "student/entropy": 0.04666216727346182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.566666666666666, + "completions/max_terminated_length": 5.566666666666666, + "completions/mean_length": 1.0550000230471293, + "completions/mean_terminated_length": 1.0550000230471293, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04832210503518582, + "epoch": 0.6015266016025519, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.067891120910645, + "kd/policy_loss": -1.0734388311704, + "kd/rkl_advantage_mean": 40.902838150660195, + "kd/rkl_advantage_p95": 56.166669662793474, + "kd/rkl_advantage_std": 25.087576165795326, + "kd/ssd_loss": 0.0, + "learning_rate": 3.985113735616906e-07, + "loss": -4.293754069010417, + "num_tokens": 189308205.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0302766799926757, + "sampling/importance_sampling_ratio/mean": 0.9998448848724365, + "sampling/importance_sampling_ratio/min": 0.9342863420645396, + "sampling/sampling_logp_difference/max": 0.0811348187426726, + "sampling/sampling_logp_difference/mean": 0.001125504550388238, + "step": 15840, + "step_time": 4.674948130187113, + "student/entropy": 0.04832210503518582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.266666666666667, + "completions/max_terminated_length": 7.266666666666667, + "completions/mean_length": 1.093333367506663, + "completions/mean_terminated_length": 1.093333367506663, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06108251620704929, + "epoch": 0.6026658565298295, + "eval/gt_acc_batch": 0.0022222223691642285, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.176767349243164, + "kd/policy_loss": -4.169236421585083, + "kd/rkl_advantage_mean": 62.49742762247721, + "kd/rkl_advantage_p95": 79.23670867284139, + "kd/rkl_advantage_std": 29.741393274068834, + "kd/ssd_loss": 0.0, + "learning_rate": 3.9737211863441304e-07, + "loss": -16.676944986979166, + "num_tokens": 189613485.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0022222223070760567, + "rewards/gt_logging_reward/std": 0.020440502216418584, + "sampling/importance_sampling_ratio/max": 1.0297764817873636, + "sampling/importance_sampling_ratio/mean": 1.0001449684302013, + "sampling/importance_sampling_ratio/min": 0.9574898183345795, + "sampling/sampling_logp_difference/max": 0.06298079422364632, + "sampling/sampling_logp_difference/mean": 0.001246352243470028, + "step": 15870, + "step_time": 4.653015952072262, + "student/entropy": 0.06108251620704929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.733333333333333, + "completions/max_terminated_length": 7.733333333333333, + "completions/mean_length": 1.0719444751739502, + "completions/mean_terminated_length": 1.0719444751739502, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.053795450118680796, + "epoch": 0.603805111457107, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.945612907409668, + "kd/policy_loss": -3.121384298801422, + "kd/rkl_advantage_mean": 68.38890679677327, + "kd/rkl_advantage_p95": 90.49165740013123, + "kd/rkl_advantage_std": 37.6250980257988, + "kd/ssd_loss": 0.0, + "learning_rate": 3.962328637071355e-07, + "loss": -12.48554178873698, + "num_tokens": 189929152.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0332482179005942, + "sampling/importance_sampling_ratio/mean": 0.9999571561813354, + "sampling/importance_sampling_ratio/min": 0.9406238277753194, + "sampling/sampling_logp_difference/max": 0.08433953345132371, + "sampling/sampling_logp_difference/mean": 0.001539172947135133, + "step": 15900, + "step_time": 4.819100208285575, + "student/entropy": 0.053795450118680796 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.233333333333333, + "completions/max_terminated_length": 6.233333333333333, + "completions/mean_length": 1.0561111370722454, + "completions/mean_terminated_length": 1.0561111370722454, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04415510650724173, + "epoch": 0.6049443663843846, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.545703887939453, + "kd/policy_loss": -1.2857582251230875, + "kd/rkl_advantage_mean": 49.44742890993754, + "kd/rkl_advantage_p95": 67.67895251115164, + "kd/rkl_advantage_std": 30.612508778770763, + "kd/ssd_loss": 0.0, + "learning_rate": 3.95093608779858e-07, + "loss": -5.14303232828776, + "num_tokens": 190241514.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0271775682767232, + "sampling/importance_sampling_ratio/mean": 1.0000044544537863, + "sampling/importance_sampling_ratio/min": 0.9646503825982412, + "sampling/sampling_logp_difference/max": 0.05168453052950402, + "sampling/sampling_logp_difference/mean": 0.0008721779386784571, + "step": 15930, + "step_time": 4.6836508166238975, + "student/entropy": 0.04415510650724173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.933333333333334, + "completions/max_terminated_length": 8.933333333333334, + "completions/mean_length": 1.0961111426353454, + "completions/mean_terminated_length": 1.0961111426353454, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04883764963597059, + "epoch": 0.6060836213116622, + "eval/gt_acc_batch": 0.0027777779226501784, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.474092483520508, + "kd/policy_loss": -5.192741584777832, + "kd/rkl_advantage_mean": 79.8471099058787, + "kd/rkl_advantage_p95": 101.92310916582743, + "kd/rkl_advantage_std": 38.034384379784264, + "kd/ssd_loss": 0.0, + "learning_rate": 3.939543538525804e-07, + "loss": -20.770965576171875, + "num_tokens": 190550852.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0027777779226501784, + "rewards/gt_logging_reward/std": 0.026827810704708098, + "sampling/importance_sampling_ratio/max": 1.0542571425437928, + "sampling/importance_sampling_ratio/mean": 1.0001595536867778, + "sampling/importance_sampling_ratio/min": 0.9509899377822876, + "sampling/sampling_logp_difference/max": 0.10854654863942415, + "sampling/sampling_logp_difference/mean": 0.0015565304221430174, + "step": 15960, + "step_time": 4.748906008949659, + "student/entropy": 0.04883764963597059 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.4, + "completions/max_terminated_length": 5.4, + "completions/mean_length": 1.0452777942021687, + "completions/mean_terminated_length": 1.0452777942021687, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04077796631803115, + "epoch": 0.6072228762389398, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.062568664550781, + "kd/policy_loss": -0.132561461130778, + "kd/rkl_advantage_mean": 35.915734688440956, + "kd/rkl_advantage_p95": 50.43836204210917, + "kd/rkl_advantage_std": 24.142923254768053, + "kd/ssd_loss": 0.0, + "learning_rate": 3.9281509892530287e-07, + "loss": -0.5302466710408529, + "num_tokens": 190866743.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0228970011075338, + "sampling/importance_sampling_ratio/mean": 1.0000630835692088, + "sampling/importance_sampling_ratio/min": 0.9718624711036682, + "sampling/sampling_logp_difference/max": 0.04698098093116035, + "sampling/sampling_logp_difference/mean": 0.0006968335035101821, + "step": 15990, + "step_time": 4.794271983000605, + "student/entropy": 0.04077796631803115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.166666666666667, + "completions/max_terminated_length": 7.166666666666667, + "completions/mean_length": 1.079722253481547, + "completions/mean_terminated_length": 1.079722253481547, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05306013928105434, + "epoch": 0.6083621311662173, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.081487655639648, + "kd/policy_loss": -3.888617912928263, + "kd/rkl_advantage_mean": 64.45364669164022, + "kd/rkl_advantage_p95": 84.82594313621522, + "kd/rkl_advantage_std": 33.524462232987084, + "kd/ssd_loss": 0.0, + "learning_rate": 3.916758439980253e-07, + "loss": -15.554470825195313, + "num_tokens": 191171878.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0957308848698935, + "sampling/importance_sampling_ratio/mean": 1.0007470230261484, + "sampling/importance_sampling_ratio/min": 0.9657218674818675, + "sampling/sampling_logp_difference/max": 0.1015206093279024, + "sampling/sampling_logp_difference/mean": 0.0015422775759361684, + "step": 16020, + "step_time": 4.716535758587997, + "student/entropy": 0.05306013928105434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.4, + "completions/max_terminated_length": 7.4, + "completions/mean_length": 1.0777778069178263, + "completions/mean_terminated_length": 1.0777778069178263, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047865307641526064, + "epoch": 0.6095013860934948, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 38.55857849121094, + "kd/policy_loss": -3.2720022598902383, + "kd/rkl_advantage_mean": 64.08753496805826, + "kd/rkl_advantage_p95": 84.94902726014455, + "kd/rkl_advantage_std": 34.42472091962894, + "kd/ssd_loss": 0.0, + "learning_rate": 3.905365890707477e-07, + "loss": -13.088008626302083, + "num_tokens": 191476782.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0395388325055441, + "sampling/importance_sampling_ratio/mean": 1.0001078049341838, + "sampling/importance_sampling_ratio/min": 0.9579846938451131, + "sampling/sampling_logp_difference/max": 0.06138486561054985, + "sampling/sampling_logp_difference/mean": 0.0011366035333291316, + "step": 16050, + "step_time": 4.5414633856349, + "student/entropy": 0.047865307641526064 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.933333333333334, + "completions/max_terminated_length": 5.933333333333334, + "completions/mean_length": 1.0519444664319357, + "completions/mean_terminated_length": 1.0519444664319357, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04199598114937544, + "epoch": 0.6106406410207724, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.618481636047363, + "kd/policy_loss": -1.1058943847815195, + "kd/rkl_advantage_mean": 47.29732812245687, + "kd/rkl_advantage_p95": 67.35812941392263, + "kd/rkl_advantage_std": 31.138874350984892, + "kd/ssd_loss": 0.0, + "learning_rate": 3.8939733414347017e-07, + "loss": -4.423577880859375, + "num_tokens": 191777929.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0131518403689066, + "sampling/importance_sampling_ratio/mean": 0.9995974004268646, + "sampling/importance_sampling_ratio/min": 0.9379017512003581, + "sampling/sampling_logp_difference/max": 0.08113671427903076, + "sampling/sampling_logp_difference/mean": 0.001120431210923319, + "step": 16080, + "step_time": 4.479319584715025, + "student/entropy": 0.04199598114937544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.733333333333333, + "completions/max_terminated_length": 6.733333333333333, + "completions/mean_length": 1.0652778108914693, + "completions/mean_terminated_length": 1.0652778108914693, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05234423934792479, + "epoch": 0.6117798959480499, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.380721092224121, + "kd/policy_loss": -2.621269679069519, + "kd/rkl_advantage_mean": 58.64979152679443, + "kd/rkl_advantage_p95": 79.03848767280579, + "kd/rkl_advantage_std": 33.45948920249939, + "kd/ssd_loss": 0.0, + "learning_rate": 3.882580792161926e-07, + "loss": -10.485078938802083, + "num_tokens": 192080996.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.030729599793752, + "sampling/importance_sampling_ratio/mean": 0.9995625178019206, + "sampling/importance_sampling_ratio/min": 0.9201553523540497, + "sampling/sampling_logp_difference/max": 0.09172082107203702, + "sampling/sampling_logp_difference/mean": 0.0013349229159454505, + "step": 16110, + "step_time": 4.492342767398805, + "student/entropy": 0.05234423934792479 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.133333333333333, + "completions/max_terminated_length": 8.133333333333333, + "completions/mean_length": 1.0802778085072835, + "completions/mean_terminated_length": 1.0802778085072835, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04861397594213486, + "epoch": 0.6129191508753276, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.068632125854492, + "kd/policy_loss": -3.1296444495519, + "kd/rkl_advantage_mean": 72.36560966173808, + "kd/rkl_advantage_p95": 95.68080577055613, + "kd/rkl_advantage_std": 40.46526619096597, + "kd/ssd_loss": 0.0, + "learning_rate": 3.87118824288915e-07, + "loss": -12.518577067057292, + "num_tokens": 192389165.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0607169032096864, + "sampling/importance_sampling_ratio/mean": 0.9998689393202463, + "sampling/importance_sampling_ratio/min": 0.9224685867627461, + "sampling/sampling_logp_difference/max": 0.11668759586755187, + "sampling/sampling_logp_difference/mean": 0.001487763942956614, + "step": 16140, + "step_time": 4.505590490158648, + "student/entropy": 0.04861397594213486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.166666666666667, + "completions/max_terminated_length": 6.166666666666667, + "completions/mean_length": 1.046666685740153, + "completions/mean_terminated_length": 1.046666685740153, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0407595072252055, + "epoch": 0.6140584058026051, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.861878395080566, + "kd/policy_loss": -1.2207140882809957, + "kd/rkl_advantage_mean": 49.48783535957337, + "kd/rkl_advantage_p95": 67.53210174242655, + "kd/rkl_advantage_std": 30.894782588879266, + "kd/ssd_loss": 0.0, + "learning_rate": 3.859795693616374e-07, + "loss": -4.8828582763671875, + "num_tokens": 192686109.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0160793741544087, + "sampling/importance_sampling_ratio/mean": 0.9998421986897786, + "sampling/importance_sampling_ratio/min": 0.9601873000462849, + "sampling/sampling_logp_difference/max": 0.05634897241058449, + "sampling/sampling_logp_difference/mean": 0.0007950953372831767, + "step": 16170, + "step_time": 4.5105735027890965, + "student/entropy": 0.0407595072252055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.133333333333334, + "completions/max_terminated_length": 5.133333333333334, + "completions/mean_length": 1.0344444632530212, + "completions/mean_terminated_length": 1.0344444632530212, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04722085331256191, + "epoch": 0.6151976607298827, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.977828979492188, + "kd/policy_loss": 0.21001459360122682, + "kd/rkl_advantage_mean": 38.07556649843852, + "kd/rkl_advantage_p95": 55.7910257657369, + "kd/rkl_advantage_std": 28.3908844123284, + "kd/ssd_loss": 0.0, + "learning_rate": 3.8484031443435995e-07, + "loss": 0.840057373046875, + "num_tokens": 192991345.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0494372328122457, + "sampling/importance_sampling_ratio/mean": 1.0004577974478404, + "sampling/importance_sampling_ratio/min": 0.9906790514787038, + "sampling/sampling_logp_difference/max": 0.050574198140141864, + "sampling/sampling_logp_difference/mean": 0.0008262876275693998, + "step": 16200, + "step_time": 4.514354390507409, + "student/entropy": 0.04722085331256191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.233333333333333, + "completions/max_terminated_length": 8.233333333333333, + "completions/mean_length": 1.084166693687439, + "completions/mean_terminated_length": 1.084166693687439, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.055342637468129396, + "epoch": 0.6163369156571602, + "eval/gt_acc_batch": 0.0025000001303851606, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.34638214111328, + "kd/policy_loss": -3.48438290754954, + "kd/rkl_advantage_mean": 73.3150874932607, + "kd/rkl_advantage_p95": 96.37185044288636, + "kd/rkl_advantage_std": 39.785193242132664, + "kd/ssd_loss": 0.0, + "learning_rate": 3.8370105950708237e-07, + "loss": -13.937531534830729, + "num_tokens": 193298256.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000001303851606, + "rewards/gt_logging_reward/std": 0.025585518529017765, + "sampling/importance_sampling_ratio/max": 1.050850248336792, + "sampling/importance_sampling_ratio/mean": 1.0001911739508311, + "sampling/importance_sampling_ratio/min": 0.9507867356141408, + "sampling/sampling_logp_difference/max": 0.08309153572966656, + "sampling/sampling_logp_difference/mean": 0.0012577400513691828, + "step": 16230, + "step_time": 4.542911128063376, + "student/entropy": 0.055342637468129396 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0530555804570516, + "completions/mean_terminated_length": 1.0530555804570516, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04368698848411441, + "epoch": 0.6174761705844378, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.478276252746582, + "kd/policy_loss": -1.734591410557429, + "kd/rkl_advantage_mean": 61.40428859392802, + "kd/rkl_advantage_p95": 84.5194218715032, + "kd/rkl_advantage_std": 38.26776904463768, + "kd/ssd_loss": 0.0, + "learning_rate": 3.825618045798048e-07, + "loss": -6.938365681966146, + "num_tokens": 193592359.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0170827507972717, + "sampling/importance_sampling_ratio/mean": 0.9997242430845896, + "sampling/importance_sampling_ratio/min": 0.9430677771568299, + "sampling/sampling_logp_difference/max": 0.0672769653222834, + "sampling/sampling_logp_difference/mean": 0.0009284783833815405, + "step": 16260, + "step_time": 4.384871668725585, + "student/entropy": 0.04368698848411441 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.833333333333333, + "completions/max_terminated_length": 4.833333333333333, + "completions/mean_length": 1.043333351612091, + "completions/mean_terminated_length": 1.043333351612091, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04722962842012445, + "epoch": 0.6186154255117153, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.583982467651367, + "kd/policy_loss": -0.012991372744242351, + "kd/rkl_advantage_mean": 31.33479234377543, + "kd/rkl_advantage_p95": 44.83853340148926, + "kd/rkl_advantage_std": 21.853409507870673, + "kd/ssd_loss": 0.0, + "learning_rate": 3.8142254965252725e-07, + "loss": -0.05196524858474731, + "num_tokens": 193905091.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0148467381795248, + "sampling/importance_sampling_ratio/mean": 0.9998794813950856, + "sampling/importance_sampling_ratio/min": 0.962172782421112, + "sampling/sampling_logp_difference/max": 0.05046014714365204, + "sampling/sampling_logp_difference/mean": 0.0010308417734146739, + "step": 16290, + "step_time": 4.562112174664313, + "student/entropy": 0.04722962842012445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.0813889185587564, + "completions/mean_terminated_length": 1.0813889185587564, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05961656393483281, + "epoch": 0.6197546804389928, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 59.47653579711914, + "kd/policy_loss": -3.3847963571548463, + "kd/rkl_advantage_mean": 56.49150218963623, + "kd/rkl_advantage_p95": 73.30074515342713, + "kd/rkl_advantage_std": 28.39603722790877, + "kd/ssd_loss": 0.0, + "learning_rate": 3.8028329472524967e-07, + "loss": -13.5391845703125, + "num_tokens": 194214344.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0146270950635274, + "sampling/importance_sampling_ratio/mean": 0.9997655689716339, + "sampling/importance_sampling_ratio/min": 0.950359316666921, + "sampling/sampling_logp_difference/max": 0.05519126863218844, + "sampling/sampling_logp_difference/mean": 0.001134583393771512, + "step": 16320, + "step_time": 4.394204515152766, + "student/entropy": 0.05961656393483281 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.666666666666667, + "completions/max_terminated_length": 7.666666666666667, + "completions/mean_length": 1.0752778053283691, + "completions/mean_terminated_length": 1.0752778053283691, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05151207723344366, + "epoch": 0.6208939353662705, + "eval/gt_acc_batch": 0.0025000001303851606, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.978376388549805, + "kd/policy_loss": -2.7694049874941506, + "kd/rkl_advantage_mean": 63.851054430007935, + "kd/rkl_advantage_p95": 84.69517558415731, + "kd/rkl_advantage_std": 35.37137425243854, + "kd/ssd_loss": 0.0, + "learning_rate": 3.791440397979721e-07, + "loss": -11.07761942545573, + "num_tokens": 194521479.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000001303851606, + "rewards/gt_logging_reward/std": 0.027386129647493363, + "sampling/importance_sampling_ratio/max": 1.0247772971789042, + "sampling/importance_sampling_ratio/mean": 0.9996082643667857, + "sampling/importance_sampling_ratio/min": 0.9338128884633382, + "sampling/sampling_logp_difference/max": 0.0952972627710551, + "sampling/sampling_logp_difference/mean": 0.001337073576481392, + "step": 16350, + "step_time": 4.546792425902095, + "student/entropy": 0.05151207723344366 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.266666666666667, + "completions/max_terminated_length": 7.266666666666667, + "completions/mean_length": 1.0719444751739502, + "completions/mean_terminated_length": 1.0719444751739502, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04212590791285038, + "epoch": 0.622033190293548, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 8.987078666687012, + "kd/policy_loss": -3.3541978160540262, + "kd/rkl_advantage_mean": 64.48638761838278, + "kd/rkl_advantage_p95": 84.51752727031707, + "kd/rkl_advantage_std": 34.308407473564145, + "kd/ssd_loss": 0.0, + "learning_rate": 3.7800478487069456e-07, + "loss": -13.416788736979166, + "num_tokens": 194830578.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0143378337224325, + "sampling/importance_sampling_ratio/mean": 0.9996409515539805, + "sampling/importance_sampling_ratio/min": 0.9321308930714926, + "sampling/sampling_logp_difference/max": 0.0770929949125275, + "sampling/sampling_logp_difference/mean": 0.001025256491266191, + "step": 16380, + "step_time": 4.594566939988484, + "student/entropy": 0.04212590791285038 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.966666666666667, + "completions/max_terminated_length": 6.966666666666667, + "completions/mean_length": 1.063611133893331, + "completions/mean_terminated_length": 1.063611133893331, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.042189594746256866, + "epoch": 0.6231724452208256, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.882145881652832, + "kd/policy_loss": -2.6165380915006002, + "kd/rkl_advantage_mean": 59.068139457702635, + "kd/rkl_advantage_p95": 78.96058698495229, + "kd/rkl_advantage_std": 33.199644743899505, + "kd/ssd_loss": 0.0, + "learning_rate": 3.76865529943417e-07, + "loss": -10.466151936848958, + "num_tokens": 195137247.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0361411650975545, + "sampling/importance_sampling_ratio/mean": 0.9997960706551869, + "sampling/importance_sampling_ratio/min": 0.9288848181565602, + "sampling/sampling_logp_difference/max": 0.07990399762056768, + "sampling/sampling_logp_difference/mean": 0.0011751236277632415, + "step": 16410, + "step_time": 4.527954873535782, + "student/entropy": 0.042189594746256866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.966666666666667, + "completions/max_terminated_length": 5.966666666666667, + "completions/mean_length": 1.0561111410458883, + "completions/mean_terminated_length": 1.0561111410458883, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04399045507113139, + "epoch": 0.6243117001481031, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.100648880004883, + "kd/policy_loss": -1.180846114953359, + "kd/rkl_advantage_mean": 48.31958131790161, + "kd/rkl_advantage_p95": 67.6729700088501, + "kd/rkl_advantage_std": 30.90099381605784, + "kd/ssd_loss": 0.0, + "learning_rate": 3.757262750161394e-07, + "loss": -4.7233835856119795, + "num_tokens": 195442073.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0091963569323221, + "sampling/importance_sampling_ratio/mean": 0.9998868266741435, + "sampling/importance_sampling_ratio/min": 0.9624858240286509, + "sampling/sampling_logp_difference/max": 0.04752147564043601, + "sampling/sampling_logp_difference/mean": 0.0008361732616322115, + "step": 16440, + "step_time": 4.517827322291366, + "student/entropy": 0.04399045507113139 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.433333333333334, + "completions/max_terminated_length": 7.433333333333334, + "completions/mean_length": 1.0780555804570515, + "completions/mean_terminated_length": 1.0780555804570515, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04145172929080824, + "epoch": 0.6254509550753807, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 8.785391807556152, + "kd/policy_loss": -3.2960206111272177, + "kd/rkl_advantage_mean": 64.71621446609497, + "kd/rkl_advantage_p95": 84.58107785383861, + "kd/rkl_advantage_std": 34.19664618869623, + "kd/ssd_loss": 0.0, + "learning_rate": 3.745870200888619e-07, + "loss": -13.18408203125, + "num_tokens": 195752154.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.017483619848887, + "sampling/importance_sampling_ratio/mean": 0.9996735632419587, + "sampling/importance_sampling_ratio/min": 0.9400181909402211, + "sampling/sampling_logp_difference/max": 0.07236140336220463, + "sampling/sampling_logp_difference/mean": 0.0011630768989562058, + "step": 16470, + "step_time": 4.502692085943029, + "student/entropy": 0.04145172929080824 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 3.966666666666667, + "completions/max_terminated_length": 3.966666666666667, + "completions/mean_length": 1.033055567741394, + "completions/mean_terminated_length": 1.033055567741394, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04504386354237795, + "epoch": 0.6265902100026582, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.89332389831543, + "kd/policy_loss": 1.181273055076599, + "kd/rkl_advantage_mean": 20.76558101971944, + "kd/rkl_advantage_p95": 33.36101473172506, + "kd/rkl_advantage_std": 18.766038087010383, + "kd/ssd_loss": 0.0, + "learning_rate": 3.7344776516158434e-07, + "loss": 4.7250920613606775, + "num_tokens": 196062065.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0327635725339255, + "sampling/importance_sampling_ratio/mean": 1.000354778766632, + "sampling/importance_sampling_ratio/min": 0.9828203479448955, + "sampling/sampling_logp_difference/max": 0.042167939431965354, + "sampling/sampling_logp_difference/mean": 0.0008475452418982361, + "step": 16500, + "step_time": 4.584592763827337, + "student/entropy": 0.04504386354237795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.366666666666667, + "completions/max_terminated_length": 8.366666666666667, + "completions/mean_length": 1.1016667087872822, + "completions/mean_terminated_length": 1.1016667087872822, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06495829472939173, + "epoch": 0.6277294649299359, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.01231575012207, + "kd/policy_loss": -5.962791349490484, + "kd/rkl_advantage_mean": 79.35206926663717, + "kd/rkl_advantage_p95": 101.47979316711425, + "kd/rkl_advantage_std": 36.83010666966438, + "kd/ssd_loss": 0.0, + "learning_rate": 3.7230851023430675e-07, + "loss": -23.851163736979167, + "num_tokens": 196365743.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.078491246700287, + "sampling/importance_sampling_ratio/mean": 0.9999144931634267, + "sampling/importance_sampling_ratio/min": 0.888040288289388, + "sampling/sampling_logp_difference/max": 0.15103644567231336, + "sampling/sampling_logp_difference/mean": 0.0020394527450359116, + "step": 16530, + "step_time": 4.55293104549249, + "student/entropy": 0.06495829472939173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.633333333333334, + "completions/max_terminated_length": 6.633333333333334, + "completions/mean_length": 1.0544444719950359, + "completions/mean_terminated_length": 1.0544444719950359, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.050393551712234814, + "epoch": 0.6288687198572134, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.767484664916992, + "kd/policy_loss": -2.0176894863446555, + "kd/rkl_advantage_mean": 57.448017088572186, + "kd/rkl_advantage_p95": 79.03750329017639, + "kd/rkl_advantage_std": 34.816732788085936, + "kd/ssd_loss": 0.0, + "learning_rate": 3.7116925530702917e-07, + "loss": -8.070757548014322, + "num_tokens": 196672707.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0106675267219543, + "sampling/importance_sampling_ratio/mean": 1.0001490314801533, + "sampling/importance_sampling_ratio/min": 0.984191644191742, + "sampling/sampling_logp_difference/max": 0.03171898387372494, + "sampling/sampling_logp_difference/mean": 0.0007007314779912122, + "step": 16560, + "step_time": 4.503851872628244, + "student/entropy": 0.050393551712234814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.033333333333333, + "completions/max_terminated_length": 6.033333333333333, + "completions/mean_length": 1.0575000206629435, + "completions/mean_terminated_length": 1.0575000206629435, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05111920728037755, + "epoch": 0.630007974784491, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.011226654052734, + "kd/policy_loss": -1.1841573556264242, + "kd/rkl_advantage_mean": 45.60598483085632, + "kd/rkl_advantage_p95": 61.88141891161601, + "kd/rkl_advantage_std": 27.59759092628956, + "kd/ssd_loss": 0.0, + "learning_rate": 3.7003000037975164e-07, + "loss": -4.736629740397135, + "num_tokens": 196975610.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0290545145670573, + "sampling/importance_sampling_ratio/mean": 0.9997925182183584, + "sampling/importance_sampling_ratio/min": 0.9415574232737224, + "sampling/sampling_logp_difference/max": 0.0894021585273246, + "sampling/sampling_logp_difference/mean": 0.0013605010802469526, + "step": 16590, + "step_time": 4.51221480590757, + "student/entropy": 0.05111920728037755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.033333333333333, + "completions/max_terminated_length": 8.033333333333333, + "completions/mean_length": 1.0752778013547262, + "completions/mean_terminated_length": 1.0752778013547262, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052325902661929526, + "epoch": 0.6311472297117685, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.56539535522461, + "kd/policy_loss": -2.73790754477183, + "kd/rkl_advantage_mean": 68.36045446395875, + "kd/rkl_advantage_p95": 90.43954941431682, + "kd/rkl_advantage_std": 37.952611004312836, + "kd/ssd_loss": 0.0, + "learning_rate": 3.6889074545247406e-07, + "loss": -10.951627604166667, + "num_tokens": 197295689.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.017138652006785, + "sampling/importance_sampling_ratio/mean": 1.0000442087650299, + "sampling/importance_sampling_ratio/min": 0.9576432089010875, + "sampling/sampling_logp_difference/max": 0.06076197011085848, + "sampling/sampling_logp_difference/mean": 0.0011161994547971214, + "step": 16620, + "step_time": 4.621184199734126, + "student/entropy": 0.052325902661929526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.166666666666667, + "completions/max_terminated_length": 7.166666666666667, + "completions/mean_length": 1.0625000238418578, + "completions/mean_terminated_length": 1.0625000238418578, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04777659717947245, + "epoch": 0.632286484639046, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.232315063476562, + "kd/policy_loss": -2.342135496934255, + "kd/rkl_advantage_mean": 59.24398744901021, + "kd/rkl_advantage_p95": 78.59152672290801, + "kd/rkl_advantage_std": 33.27436654468377, + "kd/ssd_loss": 0.0, + "learning_rate": 3.677514905251965e-07, + "loss": -9.36854248046875, + "num_tokens": 197600394.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0472665588061014, + "sampling/importance_sampling_ratio/mean": 1.0001881897449494, + "sampling/importance_sampling_ratio/min": 0.9547530015309652, + "sampling/sampling_logp_difference/max": 0.0732915834368517, + "sampling/sampling_logp_difference/mean": 0.0011355742855812423, + "step": 16650, + "step_time": 4.597577089777527, + "student/entropy": 0.04777659717947245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.433333333333334, + "completions/max_terminated_length": 6.433333333333334, + "completions/mean_length": 1.0758333643277487, + "completions/mean_terminated_length": 1.0758333643277487, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04468793167422215, + "epoch": 0.6334257395663236, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.241249084472656, + "kd/policy_loss": -2.4543614625930785, + "kd/rkl_advantage_mean": 51.26119081179301, + "kd/rkl_advantage_p95": 67.80581641197205, + "kd/rkl_advantage_std": 27.700848544637363, + "kd/ssd_loss": 0.0, + "learning_rate": 3.6661223559791895e-07, + "loss": -9.817445882161458, + "num_tokens": 197907979.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0365005532900493, + "sampling/importance_sampling_ratio/mean": 1.0001100897789001, + "sampling/importance_sampling_ratio/min": 0.9650400420029958, + "sampling/sampling_logp_difference/max": 0.06820613932795823, + "sampling/sampling_logp_difference/mean": 0.001198612527999406, + "step": 16680, + "step_time": 4.656213494910237, + "student/entropy": 0.04468793167422215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.8, + "completions/max_terminated_length": 7.8, + "completions/mean_length": 1.1050000429153441, + "completions/mean_terminated_length": 1.1050000429153441, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049805600506563984, + "epoch": 0.6345649944936012, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 42.34796905517578, + "kd/policy_loss": -5.218475186824799, + "kd/rkl_advantage_mean": 67.90957679748536, + "kd/rkl_advantage_p95": 85.14236617883047, + "kd/rkl_advantage_std": 30.173071083426475, + "kd/ssd_loss": 0.0, + "learning_rate": 3.6547298067064137e-07, + "loss": -20.8739013671875, + "num_tokens": 198219813.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0321481426556904, + "sampling/importance_sampling_ratio/mean": 0.999873963991801, + "sampling/importance_sampling_ratio/min": 0.9315237085024516, + "sampling/sampling_logp_difference/max": 0.08544043807002405, + "sampling/sampling_logp_difference/mean": 0.001432814645037676, + "step": 16710, + "step_time": 4.6880378467302455, + "student/entropy": 0.049805600506563984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.051111133893331, + "completions/mean_terminated_length": 1.051111133893331, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.045568695695449905, + "epoch": 0.6357042494208788, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.585874557495117, + "kd/policy_loss": -1.6014286955197652, + "kd/rkl_advantage_mean": 49.983207543691, + "kd/rkl_advantage_p95": 68.0375955661138, + "kd/rkl_advantage_std": 30.002842339873315, + "kd/ssd_loss": 0.0, + "learning_rate": 3.6433372574336384e-07, + "loss": -6.405714416503907, + "num_tokens": 198521229.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0206748326619466, + "sampling/importance_sampling_ratio/mean": 0.9999060908953349, + "sampling/importance_sampling_ratio/min": 0.9490039249261221, + "sampling/sampling_logp_difference/max": 0.06619043016495804, + "sampling/sampling_logp_difference/mean": 0.00093094481320198, + "step": 16740, + "step_time": 4.75701184310019, + "student/entropy": 0.045568695695449905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.633333333333334, + "completions/max_terminated_length": 7.633333333333334, + "completions/mean_length": 1.0791666905085247, + "completions/mean_terminated_length": 1.0791666905085247, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04422702354689439, + "epoch": 0.6368435043481563, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.502716064453125, + "kd/policy_loss": -3.3586956818898517, + "kd/rkl_advantage_mean": 65.51390234629314, + "kd/rkl_advantage_p95": 85.21333314577738, + "kd/rkl_advantage_std": 34.221053992708526, + "kd/ssd_loss": 0.0, + "learning_rate": 3.631944708160863e-07, + "loss": -13.434783935546875, + "num_tokens": 198839426.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0285630226135254, + "sampling/importance_sampling_ratio/mean": 0.9998595515886942, + "sampling/importance_sampling_ratio/min": 0.9412291506926219, + "sampling/sampling_logp_difference/max": 0.08088466625971098, + "sampling/sampling_logp_difference/mean": 0.0011288951597331713, + "step": 16770, + "step_time": 4.868964345993785, + "student/entropy": 0.04422702354689439 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.1, + "completions/max_terminated_length": 5.1, + "completions/mean_length": 1.0425000190734863, + "completions/mean_terminated_length": 1.0425000190734863, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04102755254134536, + "epoch": 0.6379827592754339, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.7022123336792, + "kd/policy_loss": 0.29820247888565066, + "kd/rkl_advantage_mean": 34.41429107983907, + "kd/rkl_advantage_p95": 50.697332406044005, + "kd/rkl_advantage_std": 25.494363753994307, + "kd/ssd_loss": 0.0, + "learning_rate": 3.620552158888087e-07, + "loss": 1.192810312906901, + "num_tokens": 199163699.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0210137287775676, + "sampling/importance_sampling_ratio/mean": 1.0000601947307586, + "sampling/importance_sampling_ratio/min": 0.9663502454757691, + "sampling/sampling_logp_difference/max": 0.03852692454432448, + "sampling/sampling_logp_difference/mean": 0.0007325148971479696, + "step": 16800, + "step_time": 4.604986201059849, + "student/entropy": 0.04102755254134536 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.058055587609609, + "completions/mean_terminated_length": 1.058055587609609, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04904520077009996, + "epoch": 0.6391220142027114, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.296859741210938, + "kd/policy_loss": -2.472599172592163, + "kd/rkl_advantage_mean": 59.395651737848915, + "kd/rkl_advantage_p95": 84.54388527870178, + "kd/rkl_advantage_std": 36.86171341141065, + "kd/ssd_loss": 0.0, + "learning_rate": 3.6091596096153114e-07, + "loss": -9.890395100911459, + "num_tokens": 199468644.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0262796044349671, + "sampling/importance_sampling_ratio/mean": 0.9999366283416748, + "sampling/importance_sampling_ratio/min": 0.9416360437870026, + "sampling/sampling_logp_difference/max": 0.11117031268465022, + "sampling/sampling_logp_difference/mean": 0.0014578142921285082, + "step": 16830, + "step_time": 4.724874015466776, + "student/entropy": 0.04904520077009996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.1, + "completions/max_terminated_length": 7.1, + "completions/mean_length": 1.0588889161745707, + "completions/mean_terminated_length": 1.0588889161745707, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04443994984030723, + "epoch": 0.640261269129989, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.585052490234375, + "kd/policy_loss": -2.7345839460690815, + "kd/rkl_advantage_mean": 63.26204543113708, + "kd/rkl_advantage_p95": 84.60741188526154, + "kd/rkl_advantage_std": 35.95988645950953, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5977670603425356e-07, + "loss": -10.938334147135416, + "num_tokens": 199771632.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0210649530092875, + "sampling/importance_sampling_ratio/mean": 1.0000617802143097, + "sampling/importance_sampling_ratio/min": 0.9635098854700724, + "sampling/sampling_logp_difference/max": 0.059795222074414296, + "sampling/sampling_logp_difference/mean": 0.0009482463331854282, + "step": 16860, + "step_time": 4.705109244061168, + "student/entropy": 0.04443994984030723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.1, + "completions/max_terminated_length": 7.1, + "completions/mean_length": 1.0588889122009277, + "completions/mean_terminated_length": 1.0588889122009277, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04171815919689834, + "epoch": 0.6414005240572666, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.645387649536133, + "kd/policy_loss": -1.7163251797358194, + "kd/rkl_advantage_mean": 58.3585919380188, + "kd/rkl_advantage_p95": 79.11294977664947, + "kd/rkl_advantage_std": 34.924245263139404, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5863745110697603e-07, + "loss": -6.865299479166667, + "num_tokens": 200072836.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0116358121236166, + "sampling/importance_sampling_ratio/mean": 0.9997364342212677, + "sampling/importance_sampling_ratio/min": 0.9423632284005483, + "sampling/sampling_logp_difference/max": 0.0685981422352294, + "sampling/sampling_logp_difference/mean": 0.0009761358999336759, + "step": 16890, + "step_time": 4.590902779790728, + "student/entropy": 0.04171815919689834 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.1, + "completions/max_terminated_length": 5.1, + "completions/mean_length": 1.048888905843099, + "completions/mean_terminated_length": 1.048888905843099, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04209419346104066, + "epoch": 0.6425397789845441, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.197723388671875, + "kd/policy_loss": -0.3252237399419149, + "kd/rkl_advantage_mean": 32.52655081748962, + "kd/rkl_advantage_p95": 45.2547566652298, + "kd/rkl_advantage_std": 20.732202037175497, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5749819617969845e-07, + "loss": -1.300896199544271, + "num_tokens": 200383580.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0071317434310914, + "sampling/importance_sampling_ratio/mean": 0.9998336911201477, + "sampling/importance_sampling_ratio/min": 0.9663103818893433, + "sampling/sampling_logp_difference/max": 0.04628642283690473, + "sampling/sampling_logp_difference/mean": 0.0008760543574074593, + "step": 16920, + "step_time": 4.807826521790897, + "student/entropy": 0.04209419346104066 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.071666693687439, + "completions/mean_terminated_length": 1.071666693687439, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04898989299933116, + "epoch": 0.6436790339118217, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.391310691833496, + "kd/policy_loss": -3.3436581055323282, + "kd/rkl_advantage_mean": 64.39113210042318, + "kd/rkl_advantage_p95": 84.83506474494934, + "kd/rkl_advantage_std": 34.21372996767362, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5635894125242087e-07, + "loss": -13.374629720052083, + "num_tokens": 200691222.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0269071181615195, + "sampling/importance_sampling_ratio/mean": 0.9999024967352549, + "sampling/importance_sampling_ratio/min": 0.9391721983750662, + "sampling/sampling_logp_difference/max": 0.0888720178976655, + "sampling/sampling_logp_difference/mean": 0.0012258930325818559, + "step": 16950, + "step_time": 4.731012052491618, + "student/entropy": 0.04898989299933116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.566666666666666, + "completions/max_terminated_length": 5.566666666666666, + "completions/mean_length": 1.0408333500226339, + "completions/mean_terminated_length": 1.0408333500226339, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.050011156996091204, + "epoch": 0.6448182888390992, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.36064338684082, + "kd/policy_loss": -0.13796194791793823, + "kd/rkl_advantage_mean": 39.621022605895995, + "kd/rkl_advantage_p95": 56.37037250995636, + "kd/rkl_advantage_std": 27.140966205795607, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5521968632514334e-07, + "loss": -0.5518477121988933, + "num_tokens": 200997089.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0162640452384948, + "sampling/importance_sampling_ratio/mean": 1.0000214020411173, + "sampling/importance_sampling_ratio/min": 0.9514767467975617, + "sampling/sampling_logp_difference/max": 0.05861764119472355, + "sampling/sampling_logp_difference/mean": 0.0009603669178128864, + "step": 16980, + "step_time": 4.886583265526375, + "student/entropy": 0.050011156996091204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.933333333333334, + "completions/max_terminated_length": 6.933333333333334, + "completions/mean_length": 1.0658333698908489, + "completions/mean_terminated_length": 1.0658333698908489, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05489153082792957, + "epoch": 0.6459575437663768, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.142799377441406, + "kd/policy_loss": -2.972620975971222, + "kd/rkl_advantage_mean": 65.37550988197327, + "kd/rkl_advantage_p95": 90.37519097328186, + "kd/rkl_advantage_std": 39.094988239804906, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5408043139786575e-07, + "loss": -11.890484619140626, + "num_tokens": 201307598.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0205338597297668, + "sampling/importance_sampling_ratio/mean": 0.9999342640240987, + "sampling/importance_sampling_ratio/min": 0.946974660952886, + "sampling/sampling_logp_difference/max": 0.07129566155684491, + "sampling/sampling_logp_difference/mean": 0.0012804953167991093, + "step": 17010, + "step_time": 4.773073135603529, + "student/entropy": 0.05489153082792957 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.566666666666666, + "completions/max_terminated_length": 8.566666666666666, + "completions/mean_length": 1.0816667000452678, + "completions/mean_terminated_length": 1.0816667000452678, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04367973233262698, + "epoch": 0.6470967986936543, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 8.690386772155762, + "kd/policy_loss": -4.571946183840434, + "kd/rkl_advantage_mean": 79.34500614802043, + "kd/rkl_advantage_p95": 101.96066563924154, + "kd/rkl_advantage_std": 40.06204795340697, + "kd/ssd_loss": 0.0, + "learning_rate": 3.529411764705882e-07, + "loss": -18.287786865234374, + "num_tokens": 201607644.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0206813216209412, + "sampling/importance_sampling_ratio/mean": 0.9997255563735962, + "sampling/importance_sampling_ratio/min": 0.9343958894411722, + "sampling/sampling_logp_difference/max": 0.10056845650542527, + "sampling/sampling_logp_difference/mean": 0.0013117619974461073, + "step": 17040, + "step_time": 4.642418967117555, + "student/entropy": 0.04367973233262698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.833333333333333, + "completions/max_terminated_length": 5.833333333333333, + "completions/mean_length": 1.0483333547910054, + "completions/mean_terminated_length": 1.0483333547910054, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.035763077562054, + "epoch": 0.648236053620932, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.85453987121582, + "kd/policy_loss": -0.8628562251726787, + "kd/rkl_advantage_mean": 44.544987265268965, + "kd/rkl_advantage_p95": 62.00386309623718, + "kd/rkl_advantage_std": 28.547357575098673, + "kd/ssd_loss": 0.0, + "learning_rate": 3.518019215433107e-07, + "loss": -3.451424153645833, + "num_tokens": 201930378.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0363840858141582, + "sampling/importance_sampling_ratio/mean": 1.0003220776716868, + "sampling/importance_sampling_ratio/min": 0.9754740536212921, + "sampling/sampling_logp_difference/max": 0.06319843622234961, + "sampling/sampling_logp_difference/mean": 0.0008577696469728834, + "step": 17070, + "step_time": 4.806174525121848, + "student/entropy": 0.035763077562054 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.666666666666667, + "completions/max_terminated_length": 6.666666666666667, + "completions/mean_length": 1.0675000230471292, + "completions/mean_terminated_length": 1.0675000230471292, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04115305746284624, + "epoch": 0.6493753085482095, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 39.57971954345703, + "kd/policy_loss": -2.2162550608317058, + "kd/rkl_advantage_mean": 54.47823411623637, + "kd/rkl_advantage_p95": 73.19174701372782, + "kd/rkl_advantage_std": 31.017961095273495, + "kd/ssd_loss": 0.0, + "learning_rate": 3.506626666160331e-07, + "loss": -8.865020751953125, + "num_tokens": 202236285.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0341662009557089, + "sampling/importance_sampling_ratio/mean": 1.0000106871128083, + "sampling/importance_sampling_ratio/min": 0.9482577661673228, + "sampling/sampling_logp_difference/max": 0.07633603913709522, + "sampling/sampling_logp_difference/mean": 0.001002069534054802, + "step": 17100, + "step_time": 4.907383223487219, + "student/entropy": 0.04115305746284624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.8, + "completions/max_terminated_length": 7.8, + "completions/mean_length": 1.0741666913032533, + "completions/mean_terminated_length": 1.0741666913032533, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.044101643127699694, + "epoch": 0.6505145634754871, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.621670722961426, + "kd/policy_loss": -3.4384666164716085, + "kd/rkl_advantage_mean": 66.22986388206482, + "kd/rkl_advantage_p95": 84.79216437339782, + "kd/rkl_advantage_std": 33.74101506173611, + "kd/ssd_loss": 0.0, + "learning_rate": 3.4952341168875553e-07, + "loss": -13.75386454264323, + "num_tokens": 202542304.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0325349688529968, + "sampling/importance_sampling_ratio/mean": 1.0001883665720621, + "sampling/importance_sampling_ratio/min": 0.9616209864616394, + "sampling/sampling_logp_difference/max": 0.05075432521601518, + "sampling/sampling_logp_difference/mean": 0.000942778317645813, + "step": 17130, + "step_time": 4.893562417838257, + "student/entropy": 0.044101643127699694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.6, + "completions/max_terminated_length": 5.6, + "completions/mean_length": 1.0644444704055787, + "completions/mean_terminated_length": 1.0644444704055787, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0400415587083747, + "epoch": 0.6516538184027646, + "eval/gt_acc_batch": 0.0016666667846341927, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.689538955688477, + "kd/policy_loss": -1.371578339735667, + "kd/rkl_advantage_mean": 41.66924624443054, + "kd/rkl_advantage_p95": 56.38659769694011, + "kd/rkl_advantage_std": 24.55891161163648, + "kd/ssd_loss": 0.0, + "learning_rate": 3.48384156761478e-07, + "loss": -5.486313883463541, + "num_tokens": 202852712.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666722546021, + "rewards/gt_logging_reward/std": 0.014354695628086726, + "sampling/importance_sampling_ratio/max": 1.014208455880483, + "sampling/importance_sampling_ratio/mean": 0.9998109598954519, + "sampling/importance_sampling_ratio/min": 0.9590256651242574, + "sampling/sampling_logp_difference/max": 0.05232066434497635, + "sampling/sampling_logp_difference/mean": 0.0008503563197640082, + "step": 17160, + "step_time": 4.76561395941535, + "student/entropy": 0.0400415587083747 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.533333333333333, + "completions/max_terminated_length": 5.533333333333333, + "completions/mean_length": 1.043055574099223, + "completions/mean_terminated_length": 1.043055574099223, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04630735733856758, + "epoch": 0.6527930733300421, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.24334716796875, + "kd/policy_loss": 0.20923529068628946, + "kd/rkl_advantage_mean": 39.01001496315003, + "kd/rkl_advantage_p95": 56.33023121356964, + "kd/rkl_advantage_std": 28.207850750287374, + "kd/ssd_loss": 0.0, + "learning_rate": 3.472449018342004e-07, + "loss": 0.8369424184163411, + "num_tokens": 203163355.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0203482270240785, + "sampling/importance_sampling_ratio/mean": 1.0001050194104513, + "sampling/importance_sampling_ratio/min": 0.9695298830668132, + "sampling/sampling_logp_difference/max": 0.05081206934216122, + "sampling/sampling_logp_difference/mean": 0.0008510800869165299, + "step": 17190, + "step_time": 4.86173017367643, + "student/entropy": 0.04630735733856758 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.9, + "completions/max_terminated_length": 6.9, + "completions/mean_length": 1.0677778045336406, + "completions/mean_terminated_length": 1.0677778045336406, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05364826343332728, + "epoch": 0.6539323282573197, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.851539611816406, + "kd/policy_loss": -2.6133619070053102, + "kd/rkl_advantage_mean": 59.488778845469156, + "kd/rkl_advantage_p95": 79.0572837750117, + "kd/rkl_advantage_std": 33.3525977452596, + "kd/ssd_loss": 0.0, + "learning_rate": 3.4610564690692284e-07, + "loss": -10.453446451822916, + "num_tokens": 203473655.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0298206686973572, + "sampling/importance_sampling_ratio/mean": 1.000047085682551, + "sampling/importance_sampling_ratio/min": 0.9414266308148702, + "sampling/sampling_logp_difference/max": 0.06880557529317836, + "sampling/sampling_logp_difference/mean": 0.001228406098865283, + "step": 17220, + "step_time": 4.801471491344273, + "student/entropy": 0.05364826343332728 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.266666666666667, + "completions/max_terminated_length": 7.266666666666667, + "completions/mean_length": 1.060277799765269, + "completions/mean_terminated_length": 1.060277799765269, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05326081815486153, + "epoch": 0.6550715831845972, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.264781951904297, + "kd/policy_loss": -1.6973531325658162, + "kd/rkl_advantage_mean": 57.86858660380046, + "kd/rkl_advantage_p95": 79.07431766986846, + "kd/rkl_advantage_std": 34.56834324002266, + "kd/ssd_loss": 0.0, + "learning_rate": 3.4496639197964525e-07, + "loss": -6.789412434895834, + "num_tokens": 203773832.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0558349212010703, + "sampling/importance_sampling_ratio/mean": 1.0005193610986074, + "sampling/importance_sampling_ratio/min": 0.9673426210880279, + "sampling/sampling_logp_difference/max": 0.07144891492789611, + "sampling/sampling_logp_difference/mean": 0.0011603682476561516, + "step": 17250, + "step_time": 4.836128960091931, + "student/entropy": 0.05326081815486153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.0688889185587565, + "completions/mean_terminated_length": 1.0688889185587565, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05061008141686519, + "epoch": 0.6562108381118749, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.904979705810547, + "kd/policy_loss": -1.799711807568868, + "kd/rkl_advantage_mean": 50.697766860326134, + "kd/rkl_advantage_p95": 67.75593050320943, + "kd/rkl_advantage_std": 29.50511844754219, + "kd/ssd_loss": 0.0, + "learning_rate": 3.438271370523677e-07, + "loss": -7.198848470052083, + "num_tokens": 204077656.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0166559894879659, + "sampling/importance_sampling_ratio/mean": 0.9999288896719615, + "sampling/importance_sampling_ratio/min": 0.9527117689450582, + "sampling/sampling_logp_difference/max": 0.061285274506856995, + "sampling/sampling_logp_difference/mean": 0.0011065280841042599, + "step": 17280, + "step_time": 4.784019052508908, + "student/entropy": 0.05061008141686519 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.666666666666667, + "completions/max_terminated_length": 7.666666666666667, + "completions/mean_length": 1.0625000278155008, + "completions/mean_terminated_length": 1.0625000278155008, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05346211856231094, + "epoch": 0.6573500930391524, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.801166534423828, + "kd/policy_loss": -2.7964696129163107, + "kd/rkl_advantage_mean": 67.71104906400045, + "kd/rkl_advantage_p95": 90.32819064458211, + "kd/rkl_advantage_std": 38.471282907327016, + "kd/ssd_loss": 0.0, + "learning_rate": 3.426878821250902e-07, + "loss": -11.185879516601563, + "num_tokens": 204375849.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0134800712267558, + "sampling/importance_sampling_ratio/mean": 0.9999056359132131, + "sampling/importance_sampling_ratio/min": 0.9444817781448365, + "sampling/sampling_logp_difference/max": 0.0640878170573463, + "sampling/sampling_logp_difference/mean": 0.0009883208617490406, + "step": 17310, + "step_time": 4.7518267341967055, + "student/entropy": 0.05346211856231094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.8, + "completions/max_terminated_length": 4.8, + "completions/mean_length": 1.0513889114061992, + "completions/mean_terminated_length": 1.0513889114061992, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04982180381193757, + "epoch": 0.65848934796643, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.789608001708984, + "kd/policy_loss": -0.8962877114613851, + "kd/rkl_advantage_mean": 32.39129773775736, + "kd/rkl_advantage_p95": 44.581342633565264, + "kd/rkl_advantage_std": 19.65904092590014, + "kd/ssd_loss": 0.0, + "learning_rate": 3.415486271978126e-07, + "loss": -3.5851509094238283, + "num_tokens": 204679042.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0297831018765768, + "sampling/importance_sampling_ratio/mean": 1.000267775853475, + "sampling/importance_sampling_ratio/min": 0.9724468290805817, + "sampling/sampling_logp_difference/max": 0.0413925402952979, + "sampling/sampling_logp_difference/mean": 0.0008684115135110914, + "step": 17340, + "step_time": 4.74162436806752, + "student/entropy": 0.04982180381193757 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.0, + "completions/max_terminated_length": 8.0, + "completions/mean_length": 1.0688889185587565, + "completions/mean_terminated_length": 1.0688889185587565, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05228018158425887, + "epoch": 0.6596286028937075, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 47.027313232421875, + "kd/policy_loss": -3.159141393502553, + "kd/rkl_advantage_mean": 68.8869892279307, + "kd/rkl_advantage_p95": 90.30869576930999, + "kd/rkl_advantage_std": 37.125438100099565, + "kd/ssd_loss": 0.0, + "learning_rate": 3.404093722705351e-07, + "loss": -12.636566162109375, + "num_tokens": 204983930.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0437100410461426, + "sampling/importance_sampling_ratio/mean": 1.0002439459164938, + "sampling/importance_sampling_ratio/min": 0.9545396049817403, + "sampling/sampling_logp_difference/max": 0.09735400998033583, + "sampling/sampling_logp_difference/mean": 0.001410014645080082, + "step": 17370, + "step_time": 4.697010239982046, + "student/entropy": 0.05228018158425887 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.5, + "completions/max_terminated_length": 7.5, + "completions/mean_length": 1.087777813275655, + "completions/mean_terminated_length": 1.087777813275655, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04956770328183969, + "epoch": 0.6607678578209851, + "eval/gt_acc_batch": 0.002222222338120143, + "frac_reward_zero_std": 1.0, + "grad_norm": 60.66367721557617, + "kd/policy_loss": -4.652560714880625, + "kd/rkl_advantage_mean": 66.8582346757253, + "kd/rkl_advantage_p95": 84.981702486674, + "kd/rkl_advantage_std": 31.51446665227413, + "kd/ssd_loss": 0.0, + "learning_rate": 3.392701173432575e-07, + "loss": -18.61024169921875, + "num_tokens": 205293006.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.002222222338120143, + "rewards/gt_logging_reward/std": 0.022542615234851838, + "sampling/importance_sampling_ratio/max": 1.0132869998613994, + "sampling/importance_sampling_ratio/mean": 0.9997535149256388, + "sampling/importance_sampling_ratio/min": 0.9308154284954071, + "sampling/sampling_logp_difference/max": 0.07317140448528031, + "sampling/sampling_logp_difference/mean": 0.001408535124695239, + "step": 17400, + "step_time": 4.60183798779035, + "student/entropy": 0.04956770328183969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.0561111330986024, + "completions/mean_terminated_length": 1.0561111330986024, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.03755402096236746, + "epoch": 0.6619071127482626, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.346923828125, + "kd/policy_loss": -1.8220938046773274, + "kd/rkl_advantage_mean": 49.51194140116374, + "kd/rkl_advantage_p95": 67.32951551278433, + "kd/rkl_advantage_std": 29.03190892537435, + "kd/ssd_loss": 0.0, + "learning_rate": 3.381308624159799e-07, + "loss": -7.288374837239584, + "num_tokens": 205601480.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0095117966334024, + "sampling/importance_sampling_ratio/mean": 0.9996999740600586, + "sampling/importance_sampling_ratio/min": 0.9353110313415527, + "sampling/sampling_logp_difference/max": 0.06584462782678505, + "sampling/sampling_logp_difference/mean": 0.0010034124687081202, + "step": 17430, + "step_time": 4.509030310693197, + "student/entropy": 0.03755402096236746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.766666666666667, + "completions/max_terminated_length": 6.766666666666667, + "completions/mean_length": 1.0644444664319357, + "completions/mean_terminated_length": 1.0644444664319357, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04142689903577169, + "epoch": 0.6630463676755401, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.522660255432129, + "kd/policy_loss": -2.616906193892161, + "kd/rkl_advantage_mean": 55.939646339416505, + "kd/rkl_advantage_p95": 73.6039661804835, + "kd/rkl_advantage_std": 30.20630819350481, + "kd/ssd_loss": 0.0, + "learning_rate": 3.369916074887024e-07, + "loss": -10.467624918619792, + "num_tokens": 205907816.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0419236818949382, + "sampling/importance_sampling_ratio/mean": 1.0001647512118021, + "sampling/importance_sampling_ratio/min": 0.9619730512301127, + "sampling/sampling_logp_difference/max": 0.054802074066052836, + "sampling/sampling_logp_difference/mean": 0.0010398174975610648, + "step": 17460, + "step_time": 4.636172961886041, + "student/entropy": 0.04142689903577169 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.966666666666667, + "completions/max_terminated_length": 4.966666666666667, + "completions/mean_length": 1.0513889074325562, + "completions/mean_terminated_length": 1.0513889074325562, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04999389971295993, + "epoch": 0.6641856226028178, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.0169677734375, + "kd/policy_loss": 0.7888270696004231, + "kd/rkl_advantage_mean": 26.83896436691284, + "kd/rkl_advantage_p95": 39.62262399196625, + "kd/rkl_advantage_std": 20.437964286406835, + "kd/ssd_loss": 0.0, + "learning_rate": 3.358523525614248e-07, + "loss": 3.155308787027995, + "num_tokens": 206222689.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.004285195469856262, + "sampling/importance_sampling_ratio/max": 1.0192015687624614, + "sampling/importance_sampling_ratio/mean": 1.0000626782576243, + "sampling/importance_sampling_ratio/min": 0.9611185391743978, + "sampling/sampling_logp_difference/max": 0.047557062562555076, + "sampling/sampling_logp_difference/mean": 0.0009923285911402975, + "step": 17490, + "step_time": 4.5560401447544185, + "student/entropy": 0.04999389971295993 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.1, + "completions/max_terminated_length": 8.1, + "completions/mean_length": 1.0866666952768962, + "completions/mean_terminated_length": 1.0866666952768962, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05434432576100032, + "epoch": 0.6653248775300953, + "eval/gt_acc_batch": 0.0019444445768992105, + "frac_reward_zero_std": 1.0, + "grad_norm": 34.806522369384766, + "kd/policy_loss": -3.4105737725893657, + "kd/rkl_advantage_mean": 68.9797043800354, + "kd/rkl_advantage_p95": 90.06073375542958, + "kd/rkl_advantage_std": 36.349857758482294, + "kd/ssd_loss": 0.0, + "learning_rate": 3.347130976341472e-07, + "loss": -13.642295328776042, + "num_tokens": 206527401.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445148110389, + "rewards/gt_logging_reward/std": 0.017397598922252656, + "sampling/importance_sampling_ratio/max": 1.0402997056643168, + "sampling/importance_sampling_ratio/mean": 1.00010338028272, + "sampling/importance_sampling_ratio/min": 0.9426671961943308, + "sampling/sampling_logp_difference/max": 0.08404724965803326, + "sampling/sampling_logp_difference/mean": 0.0015302597360763077, + "step": 17520, + "step_time": 4.617464459714635, + "student/entropy": 0.05434432576100032 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.466666666666667, + "completions/max_terminated_length": 7.466666666666667, + "completions/mean_length": 1.0811111410458882, + "completions/mean_terminated_length": 1.0811111410458882, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05316193988546729, + "epoch": 0.6664641324573729, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 44.23557662963867, + "kd/policy_loss": -4.580014046033224, + "kd/rkl_advantage_mean": 62.80609135627746, + "kd/rkl_advantage_p95": 78.920707933108, + "kd/rkl_advantage_std": 28.569530896345775, + "kd/ssd_loss": 0.0, + "learning_rate": 3.3357384270686964e-07, + "loss": -18.320052083333334, + "num_tokens": 206834221.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0229914347330729, + "sampling/importance_sampling_ratio/mean": 1.0000949223836264, + "sampling/importance_sampling_ratio/min": 0.9596174597740174, + "sampling/sampling_logp_difference/max": 0.058233655616641046, + "sampling/sampling_logp_difference/mean": 0.0011894016303510095, + "step": 17550, + "step_time": 4.578657961718273, + "student/entropy": 0.05316193988546729 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.065000025431315, + "completions/mean_terminated_length": 1.065000025431315, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049813792140533525, + "epoch": 0.6676033873846504, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.050869941711426, + "kd/policy_loss": -2.1606086095174155, + "kd/rkl_advantage_mean": 54.4799836953481, + "kd/rkl_advantage_p95": 73.29128268559774, + "kd/rkl_advantage_std": 31.38502753973007, + "kd/ssd_loss": 0.0, + "learning_rate": 3.3243458777959217e-07, + "loss": -8.642435709635416, + "num_tokens": 207139023.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.00955015818278, + "sampling/importance_sampling_ratio/mean": 0.9997472882270813, + "sampling/importance_sampling_ratio/min": 0.9385627528031667, + "sampling/sampling_logp_difference/max": 0.069009075484549, + "sampling/sampling_logp_difference/mean": 0.0013100859534461052, + "step": 17580, + "step_time": 4.641667092284964, + "student/entropy": 0.049813792140533525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.133333333333334, + "completions/max_terminated_length": 7.133333333333334, + "completions/mean_length": 1.0680555860201517, + "completions/mean_terminated_length": 1.0680555860201517, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04434503590067228, + "epoch": 0.668742642311928, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.001004219055176, + "kd/policy_loss": -3.2583006461461386, + "kd/rkl_advantage_mean": 63.78961947758992, + "kd/rkl_advantage_p95": 84.42197195688884, + "kd/rkl_advantage_std": 34.65591970831156, + "kd/ssd_loss": 0.0, + "learning_rate": 3.312953328523146e-07, + "loss": -13.033201090494792, + "num_tokens": 207450300.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0491916735967, + "sampling/importance_sampling_ratio/mean": 1.0001749873161316, + "sampling/importance_sampling_ratio/min": 0.9459668894608816, + "sampling/sampling_logp_difference/max": 0.10800802932741742, + "sampling/sampling_logp_difference/mean": 0.0014029068624950014, + "step": 17610, + "step_time": 4.653310089604929, + "student/entropy": 0.04434503590067228 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.066666666666666, + "completions/max_terminated_length": 6.066666666666666, + "completions/mean_length": 1.0611111362775167, + "completions/mean_terminated_length": 1.0611111362775167, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04600502370546262, + "epoch": 0.6698818972392055, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.36893081665039, + "kd/policy_loss": -1.1536717414855957, + "kd/rkl_advantage_mean": 45.639752594629925, + "kd/rkl_advantage_p95": 62.134812760353086, + "kd/rkl_advantage_std": 27.721041973431905, + "kd/ssd_loss": 0.0, + "learning_rate": 3.30156077925037e-07, + "loss": -4.614686584472656, + "num_tokens": 207750464.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0256685098012288, + "sampling/importance_sampling_ratio/mean": 1.000024151802063, + "sampling/importance_sampling_ratio/min": 0.9497222344080607, + "sampling/sampling_logp_difference/max": 0.06185984117910266, + "sampling/sampling_logp_difference/mean": 0.0010613910572525734, + "step": 17640, + "step_time": 4.519480975337016, + "student/entropy": 0.04600502370546262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.5, + "completions/max_terminated_length": 8.5, + "completions/mean_length": 1.0855555852254233, + "completions/mean_terminated_length": 1.0855555852254233, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05225245496258139, + "epoch": 0.6710211521664832, + "eval/gt_acc_batch": 0.0027777779226501784, + "frac_reward_zero_std": 1.0, + "grad_norm": 35.04001235961914, + "kd/policy_loss": -4.22344084183375, + "kd/rkl_advantage_mean": 77.78500868479411, + "kd/rkl_advantage_p95": 101.70618901252746, + "kd/rkl_advantage_std": 40.90379901130994, + "kd/ssd_loss": 0.0, + "learning_rate": 3.2901682299775947e-07, + "loss": -16.89376424153646, + "num_tokens": 208047124.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0027777779226501784, + "rewards/gt_logging_reward/std": 0.026827810704708098, + "sampling/importance_sampling_ratio/max": 1.049631114800771, + "sampling/importance_sampling_ratio/mean": 1.0002714931964873, + "sampling/importance_sampling_ratio/min": 0.950134688615799, + "sampling/sampling_logp_difference/max": 0.08289211980688076, + "sampling/sampling_logp_difference/mean": 0.0013119215339732666, + "step": 17670, + "step_time": 4.5422361182436966, + "student/entropy": 0.05225245496258139 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.133333333333334, + "completions/max_terminated_length": 5.133333333333334, + "completions/mean_length": 1.0569444656372071, + "completions/mean_terminated_length": 1.0569444656372071, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.045956678688526154, + "epoch": 0.6721604070937607, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.25704574584961, + "kd/policy_loss": -0.007332344849904378, + "kd/rkl_advantage_mean": 31.407425133387246, + "kd/rkl_advantage_p95": 45.074489076932274, + "kd/rkl_advantage_std": 21.42803608973821, + "kd/ssd_loss": 0.0, + "learning_rate": 3.278775680704819e-07, + "loss": -0.02932985226313273, + "num_tokens": 208344881.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0295891880989074, + "sampling/importance_sampling_ratio/mean": 1.0000125646591187, + "sampling/importance_sampling_ratio/min": 0.948153555393219, + "sampling/sampling_logp_difference/max": 0.048155950211609407, + "sampling/sampling_logp_difference/mean": 0.0009572846591860677, + "step": 17700, + "step_time": 4.584328536794056, + "student/entropy": 0.045956678688526154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.4, + "completions/max_terminated_length": 7.4, + "completions/mean_length": 1.0783333619435629, + "completions/mean_terminated_length": 1.0783333619435629, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.058646895736455915, + "epoch": 0.6732996620210382, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 54.04684829711914, + "kd/policy_loss": -3.708950658639272, + "kd/rkl_advantage_mean": 64.81088682810466, + "kd/rkl_advantage_p95": 84.7204083442688, + "kd/rkl_advantage_std": 33.69505661626657, + "kd/ssd_loss": 0.0, + "learning_rate": 3.267383131432043e-07, + "loss": -14.83580322265625, + "num_tokens": 208648123.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.014656197528044384, + "sampling/importance_sampling_ratio/max": 1.05810835758845, + "sampling/importance_sampling_ratio/mean": 1.0001613020896911, + "sampling/importance_sampling_ratio/min": 0.9311120649178822, + "sampling/sampling_logp_difference/max": 0.09058556005669137, + "sampling/sampling_logp_difference/mean": 0.001516844168266592, + "step": 17730, + "step_time": 4.631755082992216, + "student/entropy": 0.058646895736455915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.733333333333333, + "completions/max_terminated_length": 6.733333333333333, + "completions/mean_length": 1.0766666968663534, + "completions/mean_terminated_length": 1.0766666968663534, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05845188448826472, + "epoch": 0.6744389169483158, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.841991424560547, + "kd/policy_loss": -3.807682595650355, + "kd/rkl_advantage_mean": 59.90797068277995, + "kd/rkl_advantage_p95": 78.58120107650757, + "kd/rkl_advantage_std": 30.71678594450156, + "kd/ssd_loss": 0.0, + "learning_rate": 3.255990582159268e-07, + "loss": -15.230729166666666, + "num_tokens": 208956903.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0327948371569315, + "sampling/importance_sampling_ratio/mean": 0.9998988628387451, + "sampling/importance_sampling_ratio/min": 0.9430822630723318, + "sampling/sampling_logp_difference/max": 0.08677473094624778, + "sampling/sampling_logp_difference/mean": 0.0015456959275373567, + "step": 17760, + "step_time": 4.598445266733567, + "student/entropy": 0.05845188448826472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.466666666666667, + "completions/max_terminated_length": 7.466666666666667, + "completions/mean_length": 1.091111143430074, + "completions/mean_terminated_length": 1.091111143430074, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05360264554619789, + "epoch": 0.6755781718755933, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.829177856445312, + "kd/policy_loss": -4.267147620519002, + "kd/rkl_advantage_mean": 62.87734944025676, + "kd/rkl_advantage_p95": 79.23222400347392, + "kd/rkl_advantage_std": 29.389105253418286, + "kd/ssd_loss": 0.0, + "learning_rate": 3.244598032886492e-07, + "loss": -17.068593343098957, + "num_tokens": 209266967.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.01614959637324, + "sampling/importance_sampling_ratio/mean": 0.9997735679149627, + "sampling/importance_sampling_ratio/min": 0.9380880892276764, + "sampling/sampling_logp_difference/max": 0.07559026222055157, + "sampling/sampling_logp_difference/mean": 0.0014703687018482014, + "step": 17790, + "step_time": 4.566798047108265, + "student/entropy": 0.05360264554619789 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.033333333333333, + "completions/max_terminated_length": 7.033333333333333, + "completions/mean_length": 1.0666666944821677, + "completions/mean_terminated_length": 1.0666666944821677, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05063153222824136, + "epoch": 0.676717426802871, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.876907348632812, + "kd/policy_loss": -2.3090081572532655, + "kd/rkl_advantage_mean": 58.57519178390503, + "kd/rkl_advantage_p95": 79.12589100996654, + "kd/rkl_advantage_std": 33.60340164651473, + "kd/ssd_loss": 0.0, + "learning_rate": 3.233205483613716e-07, + "loss": -9.236032104492187, + "num_tokens": 209577023.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0074308037757873, + "sampling/importance_sampling_ratio/mean": 0.9997507095336914, + "sampling/importance_sampling_ratio/min": 0.9351000765959422, + "sampling/sampling_logp_difference/max": 0.06722513490046064, + "sampling/sampling_logp_difference/mean": 0.0012114588074230899, + "step": 17820, + "step_time": 4.57710640495643, + "student/entropy": 0.05063153222824136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.7, + "completions/max_terminated_length": 6.7, + "completions/mean_length": 1.0608333587646483, + "completions/mean_terminated_length": 1.0608333587646483, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05022597387433052, + "epoch": 0.6778566817301485, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.99235725402832, + "kd/policy_loss": -2.304310913880666, + "kd/rkl_advantage_mean": 55.240153058369955, + "kd/rkl_advantage_p95": 73.37065273125967, + "kd/rkl_advantage_std": 31.043090588847797, + "kd/ssd_loss": 0.0, + "learning_rate": 3.2218129343409414e-07, + "loss": -9.217242431640624, + "num_tokens": 209884330.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0174596707026164, + "sampling/importance_sampling_ratio/mean": 0.9998689452807109, + "sampling/importance_sampling_ratio/min": 0.9383427560329437, + "sampling/sampling_logp_difference/max": 0.07006877078674734, + "sampling/sampling_logp_difference/mean": 0.0010971562917499492, + "step": 17850, + "step_time": 4.473526419897098, + "student/entropy": 0.05022597387433052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.3, + "completions/max_terminated_length": 6.3, + "completions/mean_length": 1.0541666905085245, + "completions/mean_terminated_length": 1.0541666905085245, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05008483941977223, + "epoch": 0.6789959366574261, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.93463134765625, + "kd/policy_loss": -1.9330789605776468, + "kd/rkl_advantage_mean": 50.74955263137817, + "kd/rkl_advantage_p95": 67.66795721848806, + "kd/rkl_advantage_std": 29.080642787615457, + "kd/ssd_loss": 0.0, + "learning_rate": 3.2104203850681655e-07, + "loss": -7.732317097981771, + "num_tokens": 210184061.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.036565093199412, + "sampling/importance_sampling_ratio/mean": 1.0001655240853629, + "sampling/importance_sampling_ratio/min": 0.9479657212893168, + "sampling/sampling_logp_difference/max": 0.0812603374555086, + "sampling/sampling_logp_difference/mean": 0.0013118506166695928, + "step": 17880, + "step_time": 4.533366124599707, + "student/entropy": 0.05008483941977223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.4, + "completions/max_terminated_length": 5.4, + "completions/mean_length": 1.044444457689921, + "completions/mean_terminated_length": 1.044444457689921, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05149801739801963, + "epoch": 0.6801351915847036, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.93329620361328, + "kd/policy_loss": 0.2416923443476359, + "kd/rkl_advantage_mean": 35.50331239700317, + "kd/rkl_advantage_p95": 50.78325091203054, + "kd/rkl_advantage_std": 25.31176575223605, + "kd/ssd_loss": 0.0, + "learning_rate": 3.1990278357953897e-07, + "loss": 0.9667683283487956, + "num_tokens": 210491269.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0278619249661765, + "sampling/importance_sampling_ratio/mean": 1.0003197530905406, + "sampling/importance_sampling_ratio/min": 0.9728612621625264, + "sampling/sampling_logp_difference/max": 0.045858160313218835, + "sampling/sampling_logp_difference/mean": 0.0010092234617331997, + "step": 17910, + "step_time": 4.565735418904418, + "student/entropy": 0.05149801739801963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.233333333333333, + "completions/max_terminated_length": 7.233333333333333, + "completions/mean_length": 1.0719444791475932, + "completions/mean_terminated_length": 1.0719444791475932, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0590497628475229, + "epoch": 0.6812744465119812, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.68430519104004, + "kd/policy_loss": -3.635275157292684, + "kd/rkl_advantage_mean": 64.7142094930013, + "kd/rkl_advantage_p95": 84.43942703406016, + "kd/rkl_advantage_std": 33.81289572417736, + "kd/ssd_loss": 0.0, + "learning_rate": 3.187635286522614e-07, + "loss": -14.54110107421875, + "num_tokens": 210805120.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0110559542973836, + "sampling/importance_sampling_ratio/mean": 0.9999295314153035, + "sampling/importance_sampling_ratio/min": 0.9481883068879445, + "sampling/sampling_logp_difference/max": 0.055068522598594426, + "sampling/sampling_logp_difference/mean": 0.001156385176970313, + "step": 17940, + "step_time": 4.500038426322862, + "student/entropy": 0.0590497628475229 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.633333333333334, + "completions/max_terminated_length": 5.633333333333334, + "completions/mean_length": 1.0600000301996866, + "completions/mean_terminated_length": 1.0600000301996866, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054940170422196385, + "epoch": 0.6824137014392587, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.451757431030273, + "kd/policy_loss": -1.6868421157201132, + "kd/rkl_advantage_mean": 44.62233975728353, + "kd/rkl_advantage_p95": 61.70179016590119, + "kd/rkl_advantage_std": 27.022263686855634, + "kd/ssd_loss": 0.0, + "learning_rate": 3.1762427372498386e-07, + "loss": -6.747368367513021, + "num_tokens": 211108800.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0581658562024434, + "sampling/importance_sampling_ratio/mean": 1.0002427478631337, + "sampling/importance_sampling_ratio/min": 0.9324342687924703, + "sampling/sampling_logp_difference/max": 0.07979565357478956, + "sampling/sampling_logp_difference/mean": 0.0014144782743339115, + "step": 17970, + "step_time": 4.6078344780408464, + "student/entropy": 0.054940170422196385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.033333333333333, + "completions/max_terminated_length": 7.033333333333333, + "completions/mean_length": 1.0913889169692994, + "completions/mean_terminated_length": 1.0913889169692994, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0642529928435882, + "epoch": 0.6835529563665362, + "eval/gt_acc_batch": 0.003333333507180214, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.982276916503906, + "kd/policy_loss": -3.8157562375068665, + "kd/rkl_advantage_mean": 61.51938540140788, + "kd/rkl_advantage_p95": 79.4305571158727, + "kd/rkl_advantage_std": 30.30439362724622, + "kd/ssd_loss": 0.0, + "learning_rate": 3.164850187977063e-07, + "loss": -15.263026936848958, + "num_tokens": 211424849.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.003333333507180214, + "rewards/gt_logging_reward/std": 0.03471422841151555, + "sampling/importance_sampling_ratio/max": 1.0351804614067077, + "sampling/importance_sampling_ratio/mean": 1.0001115063826242, + "sampling/importance_sampling_ratio/min": 0.9431371649106344, + "sampling/sampling_logp_difference/max": 0.07631215509027242, + "sampling/sampling_logp_difference/mean": 0.0014361252270949385, + "step": 18000, + "step_time": 4.654559693027598, + "student/entropy": 0.0642529928435882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.233333333333333, + "completions/max_terminated_length": 7.233333333333333, + "completions/mean_length": 1.09222225745519, + "completions/mean_terminated_length": 1.09222225745519, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06124756500745813, + "epoch": 0.6846922112938139, + "eval/gt_acc_batch": 0.002222222338120143, + "frac_reward_zero_std": 1.0, + "grad_norm": 50.25540542602539, + "kd/policy_loss": -4.182680078347524, + "kd/rkl_advantage_mean": 62.15293032328288, + "kd/rkl_advantage_p95": 79.29299910068512, + "kd/rkl_advantage_std": 29.587517728408177, + "kd/ssd_loss": 0.0, + "learning_rate": 3.153457638704287e-07, + "loss": -16.730718994140624, + "num_tokens": 211722181.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.002222222338120143, + "rewards/gt_logging_reward/std": 0.022542615234851838, + "sampling/importance_sampling_ratio/max": 1.039236104488373, + "sampling/importance_sampling_ratio/mean": 1.0003219385941824, + "sampling/importance_sampling_ratio/min": 0.9526463866233825, + "sampling/sampling_logp_difference/max": 0.07139828186482192, + "sampling/sampling_logp_difference/mean": 0.00139084175558916, + "step": 18030, + "step_time": 4.517514725890942, + "student/entropy": 0.06124756500745813 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.2, + "completions/max_terminated_length": 8.2, + "completions/mean_length": 1.076388923327128, + "completions/mean_terminated_length": 1.076388923327128, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04967458797618747, + "epoch": 0.6858314662210914, + "eval/gt_acc_batch": 0.0027777779226501784, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.335650444030762, + "kd/policy_loss": -4.214195617039999, + "kd/rkl_advantage_mean": 80.86772259076436, + "kd/rkl_advantage_p95": 107.18074579238892, + "kd/rkl_advantage_std": 44.37740362783273, + "kd/ssd_loss": 0.0, + "learning_rate": 3.1420650894315117e-07, + "loss": -16.85678202311198, + "num_tokens": 212033456.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0027777779226501784, + "rewards/gt_logging_reward/std": 0.03042903294165929, + "sampling/importance_sampling_ratio/max": 1.0110987583796183, + "sampling/importance_sampling_ratio/mean": 0.9997237861156464, + "sampling/importance_sampling_ratio/min": 0.9253382027149201, + "sampling/sampling_logp_difference/max": 0.08577691605314612, + "sampling/sampling_logp_difference/mean": 0.0012440785047753403, + "step": 18060, + "step_time": 4.725500090975159, + "student/entropy": 0.04967458797618747 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.066666666666666, + "completions/max_terminated_length": 5.066666666666666, + "completions/mean_length": 1.0425000190734863, + "completions/mean_terminated_length": 1.0425000190734863, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.039815851704527935, + "epoch": 0.686970721148369, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.784564971923828, + "kd/policy_loss": 0.0098170538743337, + "kd/rkl_advantage_mean": 34.062630875905356, + "kd/rkl_advantage_p95": 50.38876763979594, + "kd/rkl_advantage_std": 24.528229425350826, + "kd/ssd_loss": 0.0, + "learning_rate": 3.130672540158736e-07, + "loss": 0.039269681771596274, + "num_tokens": 212352153.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0235939105351766, + "sampling/importance_sampling_ratio/mean": 1.000062088171641, + "sampling/importance_sampling_ratio/min": 0.9642413357893626, + "sampling/sampling_logp_difference/max": 0.040980450444233915, + "sampling/sampling_logp_difference/mean": 0.0008695998655942579, + "step": 18090, + "step_time": 4.696385770512279, + "student/entropy": 0.039815851704527935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.2, + "completions/max_terminated_length": 7.2, + "completions/mean_length": 1.087500023841858, + "completions/mean_terminated_length": 1.087500023841858, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0489954873919487, + "epoch": 0.6881099760756465, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.434426307678223, + "kd/policy_loss": -2.698584226767222, + "kd/rkl_advantage_mean": 56.12407271067301, + "kd/rkl_advantage_p95": 73.59704181353251, + "kd/rkl_advantage_std": 29.390104587872823, + "kd/ssd_loss": 0.0, + "learning_rate": 3.11927999088596e-07, + "loss": -10.794336954752604, + "num_tokens": 212654428.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0571764667828878, + "sampling/importance_sampling_ratio/mean": 1.000181003411611, + "sampling/importance_sampling_ratio/min": 0.9478625317414602, + "sampling/sampling_logp_difference/max": 0.08662782728982468, + "sampling/sampling_logp_difference/mean": 0.0014373733361329262, + "step": 18120, + "step_time": 4.843156175047625, + "student/entropy": 0.0489954873919487 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.066666666666666, + "completions/max_terminated_length": 7.066666666666666, + "completions/mean_length": 1.0811111450195312, + "completions/mean_terminated_length": 1.0811111450195312, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05509740794077515, + "epoch": 0.6892492310029241, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.811094284057617, + "kd/policy_loss": -3.208044155438741, + "kd/rkl_advantage_mean": 60.24871428807577, + "kd/rkl_advantage_p95": 78.9992147843043, + "kd/rkl_advantage_std": 31.872239386538663, + "kd/ssd_loss": 0.0, + "learning_rate": 3.107887441613185e-07, + "loss": -12.832177734375, + "num_tokens": 212959552.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.033363405863444, + "sampling/importance_sampling_ratio/mean": 0.9998754362265269, + "sampling/importance_sampling_ratio/min": 0.927312449614207, + "sampling/sampling_logp_difference/max": 0.09082070354682704, + "sampling/sampling_logp_difference/mean": 0.001428466178670836, + "step": 18150, + "step_time": 4.69999331740352, + "student/entropy": 0.05509740794077515 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.733333333333333, + "completions/max_terminated_length": 5.733333333333333, + "completions/mean_length": 1.0583333611488341, + "completions/mean_terminated_length": 1.0583333611488341, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047458862575391926, + "epoch": 0.6903884859302016, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 52.73386764526367, + "kd/policy_loss": -1.8148685216903686, + "kd/rkl_advantage_mean": 46.01808702150981, + "kd/rkl_advantage_p95": 62.08578849633535, + "kd/rkl_advantage_std": 26.904548953970274, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0964948923404094e-07, + "loss": -7.25947265625, + "num_tokens": 213272570.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0397208094596864, + "sampling/importance_sampling_ratio/mean": 1.0000202218691507, + "sampling/importance_sampling_ratio/min": 0.9549971044063568, + "sampling/sampling_logp_difference/max": 0.06223696872281532, + "sampling/sampling_logp_difference/mean": 0.0011758400457135092, + "step": 18180, + "step_time": 4.8937851572176445, + "student/entropy": 0.047458862575391926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.633333333333334, + "completions/max_terminated_length": 6.633333333333334, + "completions/mean_length": 1.0630555788675944, + "completions/mean_terminated_length": 1.0630555788675944, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0539846433326602, + "epoch": 0.6915277408574793, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.546189308166504, + "kd/policy_loss": -2.549912683169047, + "kd/rkl_advantage_mean": 54.812609338760375, + "kd/rkl_advantage_p95": 73.36022936503092, + "kd/rkl_advantage_std": 30.39077897568544, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0851023430676336e-07, + "loss": -10.199652099609375, + "num_tokens": 213571973.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0260567347208658, + "sampling/importance_sampling_ratio/mean": 0.9999017854531606, + "sampling/importance_sampling_ratio/min": 0.9450459738572439, + "sampling/sampling_logp_difference/max": 0.06790834477481743, + "sampling/sampling_logp_difference/mean": 0.0011447604832937942, + "step": 18210, + "step_time": 4.7450644476378026, + "student/entropy": 0.0539846433326602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.9, + "completions/max_terminated_length": 5.9, + "completions/mean_length": 1.0597222526868184, + "completions/mean_terminated_length": 1.0597222526868184, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05288983794550101, + "epoch": 0.6926669957847568, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.843164443969727, + "kd/policy_loss": -1.8060242056846618, + "kd/rkl_advantage_mean": 48.20161321957906, + "kd/rkl_advantage_p95": 67.54342974821726, + "kd/rkl_advantage_std": 29.538131896654765, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0737097937948583e-07, + "loss": -7.2240966796875, + "num_tokens": 213880756.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.019499711940685907, + "sampling/importance_sampling_ratio/max": 1.023139202594757, + "sampling/importance_sampling_ratio/mean": 1.0000925421714784, + "sampling/importance_sampling_ratio/min": 0.9585515975952148, + "sampling/sampling_logp_difference/max": 0.07341442910643915, + "sampling/sampling_logp_difference/mean": 0.00123797365425465, + "step": 18240, + "step_time": 4.841230157844256, + "student/entropy": 0.05288983794550101 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.3, + "completions/max_terminated_length": 7.3, + "completions/mean_length": 1.0847222566604615, + "completions/mean_terminated_length": 1.0847222566604615, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054843961323301, + "epoch": 0.6938062507120343, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.149203300476074, + "kd/policy_loss": -4.004311176141103, + "kd/rkl_advantage_mean": 65.95436056454976, + "kd/rkl_advantage_p95": 84.86499760945638, + "kd/rkl_advantage_std": 33.117248274882634, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0623172445220825e-07, + "loss": -16.017244466145833, + "num_tokens": 214185229.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0415047844250997, + "sampling/importance_sampling_ratio/mean": 1.000175815820694, + "sampling/importance_sampling_ratio/min": 0.9532759467760722, + "sampling/sampling_logp_difference/max": 0.08509371364489197, + "sampling/sampling_logp_difference/mean": 0.0015153637126786635, + "step": 18270, + "step_time": 4.739429228011674, + "student/entropy": 0.054843961323301 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.5, + "completions/max_terminated_length": 6.5, + "completions/mean_length": 1.0658333579699197, + "completions/mean_terminated_length": 1.0658333579699197, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052869133992741506, + "epoch": 0.6949455056393119, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.87403678894043, + "kd/policy_loss": -1.6434438268343607, + "kd/rkl_advantage_mean": 50.46570897102356, + "kd/rkl_advantage_p95": 67.90416913032531, + "kd/rkl_advantage_std": 29.5928327391545, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0509246952493067e-07, + "loss": -6.5737767537434895, + "num_tokens": 214499434.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.019499711940685907, + "sampling/importance_sampling_ratio/max": 1.0435560584068297, + "sampling/importance_sampling_ratio/mean": 1.0002380271752676, + "sampling/importance_sampling_ratio/min": 0.9542836646238962, + "sampling/sampling_logp_difference/max": 0.05675167815449337, + "sampling/sampling_logp_difference/mean": 0.0011667386444363123, + "step": 18300, + "step_time": 4.853088769665919, + "student/entropy": 0.052869133992741506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0019444445458551248, + "completions/max_length": 124.26666666666667, + "completions/max_terminated_length": 5.6, + "completions/mean_length": 2.0441667755444843, + "completions/mean_terminated_length": 1.0506279508272807, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07934979175527891, + "epoch": 0.6960847605665894, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 71.62006378173828, + "kd/policy_loss": -2.789552789926529, + "kd/rkl_advantage_mean": 82.76112081209818, + "kd/rkl_advantage_p95": 111.35314172108968, + "kd/rkl_advantage_std": 38.3071618527174, + "kd/ssd_loss": 0.0, + "learning_rate": 3.039532145976531e-07, + "loss": -11.158211263020833, + "num_tokens": 214819057.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.2898564736048381, + "sampling/importance_sampling_ratio/mean": 1.0012575129667918, + "sampling/importance_sampling_ratio/min": 0.9596640825271606, + "sampling/sampling_logp_difference/max": 0.16146112048688036, + "sampling/sampling_logp_difference/mean": 0.0024165276088751854, + "step": 18330, + "step_time": 4.6770886863334455, + "student/entropy": 0.07934979175527891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 24.0, + "completions/max_terminated_length": 6.966666666666667, + "completions/mean_length": 1.2261111656824748, + "completions/mean_terminated_length": 1.084166709582011, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07646453017368912, + "epoch": 0.697224015493867, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.829212188720703, + "kd/policy_loss": -4.500036204854648, + "kd/rkl_advantage_mean": 72.70709896087646, + "kd/rkl_advantage_p95": 101.03442854881287, + "kd/rkl_advantage_std": 39.61504856149356, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0281395967037556e-07, + "loss": -18.00014444986979, + "num_tokens": 215123455.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.014656197528044384, + "sampling/importance_sampling_ratio/max": 1.1763427138328553, + "sampling/importance_sampling_ratio/mean": 1.0004843870798747, + "sampling/importance_sampling_ratio/min": 0.9363115231196085, + "sampling/sampling_logp_difference/max": 0.19671912123449148, + "sampling/sampling_logp_difference/mean": 0.003621869067622659, + "step": 18360, + "step_time": 4.082697872299468, + "student/entropy": 0.07646453017368912 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 23.733333333333334, + "completions/max_terminated_length": 6.7, + "completions/mean_length": 1.2144444902737936, + "completions/mean_terminated_length": 1.0725000341733297, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07316306869809826, + "epoch": 0.6983632704211445, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 73.53772735595703, + "kd/policy_loss": -3.51022625764211, + "kd/rkl_advantage_mean": 56.586423603693646, + "kd/rkl_advantage_p95": 72.8901683330536, + "kd/rkl_advantage_std": 26.38266966144244, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0167470474309797e-07, + "loss": -14.04090576171875, + "num_tokens": 215431547.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0727876325448353, + "sampling/importance_sampling_ratio/mean": 0.9995103299617767, + "sampling/importance_sampling_ratio/min": 0.9422691682974498, + "sampling/sampling_logp_difference/max": 0.13683757047789794, + "sampling/sampling_logp_difference/mean": 0.0022302346197344983, + "step": 18390, + "step_time": 4.09315868926545, + "student/entropy": 0.07316306869809826 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.633333333333334, + "completions/max_terminated_length": 7.633333333333334, + "completions/mean_length": 1.0722222487131754, + "completions/mean_terminated_length": 1.0722222487131754, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07581686607251564, + "epoch": 0.6995025253484222, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 111.17538452148438, + "kd/policy_loss": -2.9371293703715007, + "kd/rkl_advantage_mean": 58.33104923566182, + "kd/rkl_advantage_p95": 78.63641410668691, + "kd/rkl_advantage_std": 31.421031911174456, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0053544981582044e-07, + "loss": -11.748518880208334, + "num_tokens": 215753543.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.1816475947697958, + "sampling/importance_sampling_ratio/mean": 1.0005042771498363, + "sampling/importance_sampling_ratio/min": 0.9466978530089061, + "sampling/sampling_logp_difference/max": 0.11681484139213959, + "sampling/sampling_logp_difference/mean": 0.0024989697713560114, + "step": 18420, + "step_time": 4.316504585396615, + "student/entropy": 0.07581686607251564 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.166666666666667, + "completions/max_terminated_length": 7.166666666666667, + "completions/mean_length": 1.0700000325838726, + "completions/mean_terminated_length": 1.0700000325838726, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.07368431566283107, + "epoch": 0.7006417802756997, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 28.344030380249023, + "kd/policy_loss": -3.438113119204839, + "kd/rkl_advantage_mean": 61.902716938654585, + "kd/rkl_advantage_p95": 84.29156477451325, + "kd/rkl_advantage_std": 34.56882653435071, + "kd/ssd_loss": 0.0, + "learning_rate": 2.993961948885429e-07, + "loss": -13.75245361328125, + "num_tokens": 216060811.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.101072472333908, + "sampling/importance_sampling_ratio/mean": 0.9997892518838246, + "sampling/importance_sampling_ratio/min": 0.9381466348965962, + "sampling/sampling_logp_difference/max": 0.1223073410646369, + "sampling/sampling_logp_difference/mean": 0.0025432394584640862, + "step": 18450, + "step_time": 4.169432456199138, + "student/entropy": 0.07368431566283107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 25.2, + "completions/max_terminated_length": 8.533333333333333, + "completions/mean_length": 1.2100000460942586, + "completions/mean_terminated_length": 1.068081267674764, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.059828235177944104, + "epoch": 0.7017810352029773, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.156250953674316, + "kd/policy_loss": -4.279138620694479, + "kd/rkl_advantage_mean": 83.70270396868388, + "kd/rkl_advantage_p95": 112.35687311490376, + "kd/rkl_advantage_std": 45.28578642010689, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9825693996126533e-07, + "loss": -17.116552734375, + "num_tokens": 216370623.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0663853685061138, + "sampling/importance_sampling_ratio/mean": 0.9998394409815471, + "sampling/importance_sampling_ratio/min": 0.9444252610206604, + "sampling/sampling_logp_difference/max": 0.11505787746670346, + "sampling/sampling_logp_difference/mean": 0.001948733273699569, + "step": 18480, + "step_time": 4.218320503697517, + "student/entropy": 0.059828235177944104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.566666666666666, + "completions/max_terminated_length": 5.566666666666666, + "completions/mean_length": 1.045555579662323, + "completions/mean_terminated_length": 1.045555579662323, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.053731272276490924, + "epoch": 0.7029202901302548, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.039902687072754, + "kd/policy_loss": -0.7103692015012105, + "kd/rkl_advantage_mean": 39.47492605845134, + "kd/rkl_advantage_p95": 56.36540326277415, + "kd/rkl_advantage_std": 25.85497852265835, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9711768503398775e-07, + "loss": -2.8414764404296875, + "num_tokens": 216684115.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.063471535841624, + "sampling/importance_sampling_ratio/mean": 1.0002217670281728, + "sampling/importance_sampling_ratio/min": 0.9759353359540304, + "sampling/sampling_logp_difference/max": 0.08776442023615043, + "sampling/sampling_logp_difference/mean": 0.00142298514401773, + "step": 18510, + "step_time": 4.19744208723326, + "student/entropy": 0.053731272276490924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.933333333333334, + "completions/max_terminated_length": 5.933333333333334, + "completions/mean_length": 1.0541666905085245, + "completions/mean_terminated_length": 1.0541666905085245, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06324664754793048, + "epoch": 0.7040595450575323, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.423051834106445, + "kd/policy_loss": -1.1525706013043722, + "kd/rkl_advantage_mean": 43.59284022649129, + "kd/rkl_advantage_p95": 62.041945576667786, + "kd/rkl_advantage_std": 27.717510758837065, + "kd/ssd_loss": 0.0, + "learning_rate": 2.959784301067102e-07, + "loss": -4.610282389322917, + "num_tokens": 216994334.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.1104725042978922, + "sampling/importance_sampling_ratio/mean": 1.0002501467863718, + "sampling/importance_sampling_ratio/min": 0.9552482088406881, + "sampling/sampling_logp_difference/max": 0.13986329802622396, + "sampling/sampling_logp_difference/mean": 0.0018741774363055204, + "step": 18540, + "step_time": 4.084286739868306, + "student/entropy": 0.06324664754793048 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 25.0, + "completions/max_terminated_length": 8.533333333333333, + "completions/mean_length": 1.2372222582499186, + "completions/mean_terminated_length": 1.0953174948692321, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06975840022787452, + "epoch": 0.7051987999848099, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 53.283023834228516, + "kd/policy_loss": -4.64728572567304, + "kd/rkl_advantage_mean": 76.45435708363851, + "kd/rkl_advantage_p95": 96.13592496713002, + "kd/rkl_advantage_std": 35.34122818112373, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9483917517943264e-07, + "loss": -18.589143880208333, + "num_tokens": 217309852.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.070251222451528, + "sampling/importance_sampling_ratio/mean": 0.9998347242673238, + "sampling/importance_sampling_ratio/min": 0.9445833146572113, + "sampling/sampling_logp_difference/max": 0.09829246178269387, + "sampling/sampling_logp_difference/mean": 0.0019254708119357626, + "step": 18570, + "step_time": 4.269062388102369, + "student/entropy": 0.06975840022787452 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.366666666666666, + "completions/max_terminated_length": 7.366666666666666, + "completions/mean_length": 1.075277809302012, + "completions/mean_terminated_length": 1.075277809302012, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06148571064695716, + "epoch": 0.7063380549120875, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.526906967163086, + "kd/policy_loss": -3.120644414424896, + "kd/rkl_advantage_mean": 59.76255521774292, + "kd/rkl_advantage_p95": 79.13811333179474, + "kd/rkl_advantage_std": 31.473231895267965, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9369992025215506e-07, + "loss": -12.482579549153646, + "num_tokens": 217615075.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0955518325169882, + "sampling/importance_sampling_ratio/mean": 1.0001533607641855, + "sampling/importance_sampling_ratio/min": 0.9410942733287812, + "sampling/sampling_logp_difference/max": 0.08744896352291107, + "sampling/sampling_logp_difference/mean": 0.0018139265099307521, + "step": 18600, + "step_time": 4.128989621665581, + "student/entropy": 0.06148571064695716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.533333333333333, + "completions/max_terminated_length": 7.533333333333333, + "completions/mean_length": 1.0697222550710042, + "completions/mean_terminated_length": 1.0697222550710042, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05970162100469072, + "epoch": 0.7074773098393651, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.992040634155273, + "kd/policy_loss": -2.3289016842842103, + "kd/rkl_advantage_mean": 62.03526517550151, + "kd/rkl_advantage_p95": 84.37510027885438, + "kd/rkl_advantage_std": 36.347585105895995, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9256066532487747e-07, + "loss": -9.315608723958333, + "num_tokens": 217915414.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.1057084560394288, + "sampling/importance_sampling_ratio/mean": 1.0002669990062714, + "sampling/importance_sampling_ratio/min": 0.9356584409872691, + "sampling/sampling_logp_difference/max": 0.11881795252362887, + "sampling/sampling_logp_difference/mean": 0.0020173797510021057, + "step": 18630, + "step_time": 4.093371458062029, + "student/entropy": 0.05970162100469072 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0005555555845300357, + "completions/max_length": 40.166666666666664, + "completions/max_terminated_length": 6.533333333333333, + "completions/mean_length": 1.354722265402476, + "completions/mean_terminated_length": 1.070863707860311, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06362663935869932, + "epoch": 0.7086165647666426, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.23973274230957, + "kd/policy_loss": -2.6002642452716827, + "kd/rkl_advantage_mean": 59.1494557539622, + "kd/rkl_advantage_p95": 79.05549048582712, + "kd/rkl_advantage_std": 30.09056425988674, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9142141039759994e-07, + "loss": -10.401056925455729, + "num_tokens": 218221731.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.086699124177297, + "sampling/importance_sampling_ratio/mean": 0.9999046285947164, + "sampling/importance_sampling_ratio/min": 0.9238189200560252, + "sampling/sampling_logp_difference/max": 0.1344918470674505, + "sampling/sampling_logp_difference/mean": 0.0019748485404610014, + "step": 18660, + "step_time": 4.239573365499382, + "student/entropy": 0.06362663935869932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.866666666666667, + "completions/max_terminated_length": 8.866666666666667, + "completions/mean_length": 1.0947222510973613, + "completions/mean_terminated_length": 1.0947222510973613, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05782277137041092, + "epoch": 0.7097558196939202, + "eval/gt_acc_batch": 0.0025000001303851606, + "frac_reward_zero_std": 1.0, + "grad_norm": 132.17636108398438, + "kd/policy_loss": -4.732150677839915, + "kd/rkl_advantage_mean": 75.79780977567037, + "kd/rkl_advantage_p95": 95.72294271787008, + "kd/rkl_advantage_std": 36.017052657405536, + "kd/ssd_loss": 0.0, + "learning_rate": 2.902821554703224e-07, + "loss": -18.92860107421875, + "num_tokens": 218519568.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000001303851606, + "rewards/gt_logging_reward/std": 0.027386129647493363, + "sampling/importance_sampling_ratio/max": 1.0655037760734558, + "sampling/importance_sampling_ratio/mean": 1.000147408246994, + "sampling/importance_sampling_ratio/min": 0.931925368309021, + "sampling/sampling_logp_difference/max": 0.09325501552472512, + "sampling/sampling_logp_difference/mean": 0.0017024526906122142, + "step": 18690, + "step_time": 4.106727516300452, + "student/entropy": 0.05782277137041092 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.9, + "completions/max_terminated_length": 7.9, + "completions/mean_length": 1.0925000389417012, + "completions/mean_terminated_length": 1.0925000389417012, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05333506638805072, + "epoch": 0.7108950746211977, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 36.30379867553711, + "kd/policy_loss": -5.11359900633494, + "kd/rkl_advantage_mean": 72.35675268173217, + "kd/rkl_advantage_p95": 95.6722961028417, + "kd/rkl_advantage_std": 35.871092962225276, + "kd/ssd_loss": 0.0, + "learning_rate": 2.8914290054304483e-07, + "loss": -20.454398600260415, + "num_tokens": 218821429.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0272191365559895, + "sampling/importance_sampling_ratio/mean": 0.9995717664559682, + "sampling/importance_sampling_ratio/min": 0.943866366147995, + "sampling/sampling_logp_difference/max": 0.0788407440452526, + "sampling/sampling_logp_difference/mean": 0.0014085680944845081, + "step": 18720, + "step_time": 4.072413539571183, + "student/entropy": 0.05333506638805072 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.8, + "completions/max_terminated_length": 4.8, + "completions/mean_length": 1.0447222391764324, + "completions/mean_terminated_length": 1.0447222391764324, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05086084958165884, + "epoch": 0.7120343295484753, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.10423469543457, + "kd/policy_loss": 0.03385309775670369, + "kd/rkl_advantage_mean": 27.84718109766642, + "kd/rkl_advantage_p95": 39.089562288920085, + "kd/rkl_advantage_std": 18.687295832236607, + "kd/ssd_loss": 0.0, + "learning_rate": 2.880036456157673e-07, + "loss": 0.1354112466176351, + "num_tokens": 219136694.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0302248199780781, + "sampling/importance_sampling_ratio/mean": 0.9999997715155283, + "sampling/importance_sampling_ratio/min": 0.9775592625141144, + "sampling/sampling_logp_difference/max": 0.0377143945855399, + "sampling/sampling_logp_difference/mean": 0.0007865429011872038, + "step": 18750, + "step_time": 4.231470817700998, + "student/entropy": 0.05086084958165884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.066666666666666, + "completions/max_terminated_length": 8.066666666666666, + "completions/mean_length": 1.0961111505826315, + "completions/mean_terminated_length": 1.0961111505826315, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06600348114346465, + "epoch": 0.7131735844757529, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.16484260559082, + "kd/policy_loss": -4.742667750517527, + "kd/rkl_advantage_mean": 74.58154753049214, + "kd/rkl_advantage_p95": 96.0122716665268, + "kd/rkl_advantage_std": 36.961508215467134, + "kd/ssd_loss": 0.0, + "learning_rate": 2.868643906884897e-07, + "loss": -18.9706787109375, + "num_tokens": 219438856.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0424625317255656, + "sampling/importance_sampling_ratio/mean": 0.9994655132293702, + "sampling/importance_sampling_ratio/min": 0.9173043171564738, + "sampling/sampling_logp_difference/max": 0.10354545187825957, + "sampling/sampling_logp_difference/mean": 0.0016813990611505385, + "step": 18780, + "step_time": 4.156432995798241, + "student/entropy": 0.06600348114346465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.566666666666666, + "completions/max_terminated_length": 5.566666666666666, + "completions/mean_length": 1.0402777949968973, + "completions/mean_terminated_length": 1.0402777949968973, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05718602143848936, + "epoch": 0.7143128394030304, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 121.84066772460938, + "kd/policy_loss": -0.14061533013979594, + "kd/rkl_advantage_mean": 38.760812028249106, + "kd/rkl_advantage_p95": 56.9468819697698, + "kd/rkl_advantage_std": 27.219465772310894, + "kd/ssd_loss": 0.0, + "learning_rate": 2.8572513576121214e-07, + "loss": -0.562462043762207, + "num_tokens": 219758673.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0148163596789042, + "sampling/importance_sampling_ratio/mean": 0.9998580594857533, + "sampling/importance_sampling_ratio/min": 0.971820616722107, + "sampling/sampling_logp_difference/max": 0.03521079917748769, + "sampling/sampling_logp_difference/mean": 0.0008333391752482082, + "step": 18810, + "step_time": 4.245711000199178, + "student/entropy": 0.05718602143848936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 26.066666666666666, + "completions/max_terminated_length": 9.333333333333334, + "completions/mean_length": 1.243333383401235, + "completions/mean_terminated_length": 1.1014309446016948, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06040735536565383, + "epoch": 0.715452094330308, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 40.363731384277344, + "kd/policy_loss": -4.990499730904897, + "kd/rkl_advantage_mean": 83.04759475390117, + "kd/rkl_advantage_p95": 107.35794235865275, + "kd/rkl_advantage_std": 40.01747061709563, + "kd/ssd_loss": 0.0, + "learning_rate": 2.845858808339346e-07, + "loss": -19.96199951171875, + "num_tokens": 220061637.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0594165603319803, + "sampling/importance_sampling_ratio/mean": 0.999525233109792, + "sampling/importance_sampling_ratio/min": 0.893139922618866, + "sampling/sampling_logp_difference/max": 0.1554583927616477, + "sampling/sampling_logp_difference/mean": 0.002279260714810031, + "step": 18840, + "step_time": 4.219065184665184, + "student/entropy": 0.06040735536565383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.2, + "completions/max_terminated_length": 7.2, + "completions/mean_length": 1.0769444704055786, + "completions/mean_terminated_length": 1.0769444704055786, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05532706597199043, + "epoch": 0.7165913492575855, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 178.46372985839844, + "kd/policy_loss": -3.047458072503408, + "kd/rkl_advantage_mean": 60.226176754633585, + "kd/rkl_advantage_p95": 78.93526832262675, + "kd/rkl_advantage_std": 32.04552907745043, + "kd/ssd_loss": 0.0, + "learning_rate": 2.83446625906657e-07, + "loss": -12.189834594726562, + "num_tokens": 220369450.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.1057182788848876, + "sampling/importance_sampling_ratio/mean": 1.0003228147824605, + "sampling/importance_sampling_ratio/min": 0.9461158772309621, + "sampling/sampling_logp_difference/max": 0.13868577229635168, + "sampling/sampling_logp_difference/mean": 0.001840045056208813, + "step": 18870, + "step_time": 4.214215546098421, + "student/entropy": 0.05532706597199043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.766666666666667, + "completions/max_terminated_length": 6.766666666666667, + "completions/mean_length": 1.0527778029441834, + "completions/mean_terminated_length": 1.0527778029441834, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052171196043491366, + "epoch": 0.7177306041848631, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.410353660583496, + "kd/policy_loss": -1.6895959575970967, + "kd/rkl_advantage_mean": 57.31544361114502, + "kd/rkl_advantage_p95": 79.0454085667928, + "kd/rkl_advantage_std": 35.469269168376925, + "kd/ssd_loss": 0.0, + "learning_rate": 2.8230737097937944e-07, + "loss": -6.758385213216146, + "num_tokens": 220679776.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0260794123013814, + "sampling/importance_sampling_ratio/mean": 0.9998784303665161, + "sampling/importance_sampling_ratio/min": 0.9640171885490417, + "sampling/sampling_logp_difference/max": 0.05101136195783814, + "sampling/sampling_logp_difference/mean": 0.0008947749340829129, + "step": 18900, + "step_time": 4.1699748641006105, + "student/entropy": 0.052171196043491366 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.933333333333334, + "completions/max_terminated_length": 6.933333333333334, + "completions/mean_length": 1.055833355585734, + "completions/mean_terminated_length": 1.055833355585734, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.053055226399252815, + "epoch": 0.7188698591121406, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.087539672851562, + "kd/policy_loss": -1.2028754870096843, + "kd/rkl_advantage_mean": 49.740987618764244, + "kd/rkl_advantage_p95": 67.42334518432617, + "kd/rkl_advantage_std": 30.31379722406467, + "kd/ssd_loss": 0.0, + "learning_rate": 2.811681160521019e-07, + "loss": -4.811502583821615, + "num_tokens": 220992825.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0485037088394165, + "sampling/importance_sampling_ratio/mean": 1.000056536992391, + "sampling/importance_sampling_ratio/min": 0.9587030251820882, + "sampling/sampling_logp_difference/max": 0.08119886584269503, + "sampling/sampling_logp_difference/mean": 0.0012554348815077296, + "step": 18930, + "step_time": 4.22857271933414, + "student/entropy": 0.053055226399252815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.866666666666666, + "completions/max_terminated_length": 5.866666666666666, + "completions/mean_length": 1.0588889161745707, + "completions/mean_terminated_length": 1.0588889161745707, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06306970221921801, + "epoch": 0.7200091140394183, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.576396942138672, + "kd/policy_loss": -0.8145909031232198, + "kd/rkl_advantage_mean": 40.95094429651896, + "kd/rkl_advantage_p95": 56.545209391911825, + "kd/rkl_advantage_std": 25.533223018050194, + "kd/ssd_loss": 0.0, + "learning_rate": 2.800288611248244e-07, + "loss": -3.2583635965983073, + "num_tokens": 221292749.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.030617622534434, + "sampling/importance_sampling_ratio/mean": 0.9998261074225108, + "sampling/importance_sampling_ratio/min": 0.9608531753222148, + "sampling/sampling_logp_difference/max": 0.06623394374425212, + "sampling/sampling_logp_difference/mean": 0.0013814365268141652, + "step": 18960, + "step_time": 4.168676788195929, + "student/entropy": 0.06306970221921801 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 25.0, + "completions/max_terminated_length": 7.966666666666667, + "completions/mean_length": 1.2311111569404602, + "completions/mean_terminated_length": 1.0891667008399963, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06327090449631215, + "epoch": 0.7211483689666958, + "eval/gt_acc_batch": 0.0025000001303851606, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.45427894592285, + "kd/policy_loss": -5.3915682127078375, + "kd/rkl_advantage_mean": 82.57298967043559, + "kd/rkl_advantage_p95": 107.14205119609832, + "kd/rkl_advantage_std": 39.849867012103395, + "kd/ssd_loss": 0.0, + "learning_rate": 2.788896061975468e-07, + "loss": -21.566274007161457, + "num_tokens": 221594349.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000001303851606, + "rewards/gt_logging_reward/std": 0.025585518529017765, + "sampling/importance_sampling_ratio/max": 1.0767484108606975, + "sampling/importance_sampling_ratio/mean": 0.9997194608052572, + "sampling/importance_sampling_ratio/min": 0.891921388109525, + "sampling/sampling_logp_difference/max": 0.14429111331701278, + "sampling/sampling_logp_difference/mean": 0.0021694302752924464, + "step": 18990, + "step_time": 4.225507090964917, + "student/entropy": 0.06327090449631215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.466666666666667, + "completions/max_terminated_length": 5.466666666666667, + "completions/mean_length": 1.0475000222524007, + "completions/mean_terminated_length": 1.0475000222524007, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05546743422746658, + "epoch": 0.7222876238939734, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.392314910888672, + "kd/policy_loss": -0.4567356546719869, + "kd/rkl_advantage_mean": 39.68383735020955, + "kd/rkl_advantage_p95": 56.172622625033064, + "kd/rkl_advantage_std": 26.523333251476288, + "kd/ssd_loss": 0.0, + "learning_rate": 2.777503512702692e-07, + "loss": -1.8269423166910808, + "num_tokens": 221906248.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0260419964790344, + "sampling/importance_sampling_ratio/mean": 0.9998554507891337, + "sampling/importance_sampling_ratio/min": 0.9596243460973104, + "sampling/sampling_logp_difference/max": 0.05205276461007694, + "sampling/sampling_logp_difference/mean": 0.0009452473372220993, + "step": 19020, + "step_time": 4.130184161065457, + "student/entropy": 0.05546743422746658 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.933333333333334, + "completions/max_terminated_length": 6.933333333333334, + "completions/mean_length": 1.0802778124809265, + "completions/mean_terminated_length": 1.0802778124809265, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06300186837712923, + "epoch": 0.7234268788212509, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.304738998413086, + "kd/policy_loss": -3.164439908663432, + "kd/rkl_advantage_mean": 59.38902293841044, + "kd/rkl_advantage_p95": 79.36333805720011, + "kd/rkl_advantage_std": 32.14510393639406, + "kd/ssd_loss": 0.0, + "learning_rate": 2.766110963429917e-07, + "loss": -12.657759602864584, + "num_tokens": 222227129.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.019499711940685907, + "sampling/importance_sampling_ratio/max": 1.0370693882306417, + "sampling/importance_sampling_ratio/mean": 0.9996057470639547, + "sampling/importance_sampling_ratio/min": 0.9429176807403564, + "sampling/sampling_logp_difference/max": 0.09623694865343471, + "sampling/sampling_logp_difference/mean": 0.0014984279259806499, + "step": 19050, + "step_time": 4.249089068303389, + "student/entropy": 0.06300186837712923 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0005555555845300357, + "completions/max_length": 41.86666666666667, + "completions/max_terminated_length": 8.533333333333333, + "completions/mean_length": 1.3883333881696065, + "completions/mean_terminated_length": 1.1045425216356912, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06312295021489263, + "epoch": 0.7245661337485284, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.3035306930542, + "kd/policy_loss": -5.215170600016912, + "kd/rkl_advantage_mean": 77.47739162445069, + "kd/rkl_advantage_p95": 96.34249748388926, + "kd/rkl_advantage_std": 32.638114707171916, + "kd/ssd_loss": 0.0, + "learning_rate": 2.754718414157141e-07, + "loss": -20.86068115234375, + "num_tokens": 222537743.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0911156415939331, + "sampling/importance_sampling_ratio/mean": 1.0000960608323415, + "sampling/importance_sampling_ratio/min": 0.9126277089118957, + "sampling/sampling_logp_difference/max": 0.13290598651704688, + "sampling/sampling_logp_difference/mean": 0.0018779388318459192, + "step": 19080, + "step_time": 4.360689472967351, + "student/entropy": 0.06312295021489263 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.071666701634725, + "completions/mean_terminated_length": 1.071666701634725, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054190746912111835, + "epoch": 0.725705388675806, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 34.326759338378906, + "kd/policy_loss": -3.7958681682745614, + "kd/rkl_advantage_mean": 65.66417617797852, + "kd/rkl_advantage_p95": 90.43815444310506, + "kd/rkl_advantage_std": 36.954531259834766, + "kd/ssd_loss": 0.0, + "learning_rate": 2.743325864884365e-07, + "loss": -15.183474731445312, + "num_tokens": 222840145.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0481310129165649, + "sampling/importance_sampling_ratio/mean": 1.0001235127449035, + "sampling/importance_sampling_ratio/min": 0.9611008644104004, + "sampling/sampling_logp_difference/max": 0.08172680710752805, + "sampling/sampling_logp_difference/mean": 0.0014148198339777689, + "step": 19110, + "step_time": 4.23092537666477, + "student/entropy": 0.054190746912111835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.133333333333334, + "completions/max_terminated_length": 4.133333333333334, + "completions/mean_length": 1.0450000166893005, + "completions/mean_terminated_length": 1.0450000166893005, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0599202640975515, + "epoch": 0.7268446436030835, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 30.02634048461914, + "kd/policy_loss": 0.52797824939092, + "kd/rkl_advantage_mean": 22.040095233917235, + "kd/rkl_advantage_p95": 33.59081350962321, + "kd/rkl_advantage_std": 17.358897883693377, + "kd/ssd_loss": 0.0, + "learning_rate": 2.73193331561159e-07, + "loss": 2.111913045247396, + "num_tokens": 223152387.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0224739332993826, + "sampling/importance_sampling_ratio/mean": 0.9997361799081167, + "sampling/importance_sampling_ratio/min": 0.9635029176870982, + "sampling/sampling_logp_difference/max": 0.05630067794894179, + "sampling/sampling_logp_difference/mean": 0.0011635724406611795, + "step": 19140, + "step_time": 4.1652249795355605, + "student/entropy": 0.0599202640975515 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.533333333333333, + "completions/max_terminated_length": 8.533333333333333, + "completions/mean_length": 1.0936111569404603, + "completions/mean_terminated_length": 1.0936111569404603, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06731255011012156, + "epoch": 0.7279838985303612, + "eval/gt_acc_batch": 0.003333333507180214, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.649150848388672, + "kd/policy_loss": -5.687333699067434, + "kd/rkl_advantage_mean": 83.19948852856955, + "kd/rkl_advantage_p95": 107.66187872886658, + "kd/rkl_advantage_std": 41.0638236562411, + "kd/ssd_loss": 0.0, + "learning_rate": 2.720540766338814e-07, + "loss": -22.749338785807293, + "num_tokens": 223456340.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.003333333507180214, + "rewards/gt_logging_reward/std": 0.03291361729303996, + "sampling/importance_sampling_ratio/max": 1.0308601180712382, + "sampling/importance_sampling_ratio/mean": 0.9995059470335642, + "sampling/importance_sampling_ratio/min": 0.9353663941224416, + "sampling/sampling_logp_difference/max": 0.07983145797625184, + "sampling/sampling_logp_difference/mean": 0.0015066094065938765, + "step": 19170, + "step_time": 4.210925466167585, + "student/entropy": 0.06731255011012156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 24.166666666666668, + "completions/max_terminated_length": 7.533333333333333, + "completions/mean_length": 1.2136111497879027, + "completions/mean_terminated_length": 1.0716947118441265, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05629337852199872, + "epoch": 0.7291231534576387, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.286598205566406, + "kd/policy_loss": -3.5431867440541587, + "kd/rkl_advantage_mean": 65.392342821757, + "kd/rkl_advantage_p95": 84.88401579856873, + "kd/rkl_advantage_std": 32.25692260464032, + "kd/ssd_loss": 0.0, + "learning_rate": 2.7091482170660383e-07, + "loss": -14.172745768229166, + "num_tokens": 223754037.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0705011049906412, + "sampling/importance_sampling_ratio/mean": 0.9994683663050333, + "sampling/importance_sampling_ratio/min": 0.8980656097332637, + "sampling/sampling_logp_difference/max": 0.1674135471849392, + "sampling/sampling_logp_difference/mean": 0.0019455320842098444, + "step": 19200, + "step_time": 4.261408396499367, + "student/entropy": 0.05629337852199872 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.061494280552042, + "completions/mean_terminated_length": 1.061494280552042, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04977298733489267, + "epoch": 0.7302624083849163, + "eval/gt_acc_batch": 0.001149425347303522, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.6790828704834, + "kd/policy_loss": -1.8365341157748782, + "kd/rkl_advantage_mean": 54.48561841043933, + "kd/rkl_advantage_p95": 75.99437417655156, + "kd/rkl_advantage_std": 33.113237057266566, + "kd/ssd_loss": 0.0, + "learning_rate": 2.6977556677932636e-07, + "loss": -7.101265462239583, + "num_tokens": 224045139.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001149425347303522, + "rewards/gt_logging_reward/std": 0.010728622818815297, + "sampling/importance_sampling_ratio/max": 1.0519014514725784, + "sampling/importance_sampling_ratio/mean": 1.00018689961269, + "sampling/importance_sampling_ratio/min": 0.9710952906773008, + "sampling/sampling_logp_difference/max": 0.09176115524666063, + "sampling/sampling_logp_difference/mean": 0.0014079946348170653, + "step": 19230, + "step_time": 4.112896897267395, + "student/entropy": 0.04977298733489267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 23.133333333333333, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.196666701634725, + "completions/mean_terminated_length": 1.054722245534261, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05138197370494405, + "epoch": 0.7314016633121938, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.3934326171875, + "kd/policy_loss": -1.4209041357040406, + "kd/rkl_advantage_mean": 54.45613638559977, + "kd/rkl_advantage_p95": 73.31439278920492, + "kd/rkl_advantage_std": 31.484623621900877, + "kd/ssd_loss": 0.0, + "learning_rate": 2.6863631185204877e-07, + "loss": -5.683617146809896, + "num_tokens": 224355343.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0448771278063456, + "sampling/importance_sampling_ratio/mean": 1.0000630656878153, + "sampling/importance_sampling_ratio/min": 0.9589117050170899, + "sampling/sampling_logp_difference/max": 0.0660350981168449, + "sampling/sampling_logp_difference/mean": 0.0010534436296438798, + "step": 19260, + "step_time": 4.311510208332523, + "student/entropy": 0.05138197370494405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.4, + "completions/max_terminated_length": 7.4, + "completions/mean_length": 1.0794444759686788, + "completions/mean_terminated_length": 1.0794444759686788, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052659814991056916, + "epoch": 0.7325409182394714, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.625080108642578, + "kd/policy_loss": -3.2363828281561533, + "kd/rkl_advantage_mean": 59.44244062105815, + "kd/rkl_advantage_p95": 79.13220586776734, + "kd/rkl_advantage_std": 31.41499074002107, + "kd/ssd_loss": 0.0, + "learning_rate": 2.674970569247712e-07, + "loss": -12.9455322265625, + "num_tokens": 224661229.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0624444007873535, + "sampling/importance_sampling_ratio/mean": 1.0003616293271382, + "sampling/importance_sampling_ratio/min": 0.975148202975591, + "sampling/sampling_logp_difference/max": 0.0783671014631788, + "sampling/sampling_logp_difference/mean": 0.0013298987129625555, + "step": 19290, + "step_time": 4.124643843364902, + "student/entropy": 0.052659814991056916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.666666666666667, + "completions/max_terminated_length": 7.666666666666667, + "completions/mean_length": 1.0708333651224773, + "completions/mean_terminated_length": 1.0708333651224773, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04955050641049941, + "epoch": 0.7336801731667489, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 35.208866119384766, + "kd/policy_loss": -3.285124123096466, + "kd/rkl_advantage_mean": 63.482190656661984, + "kd/rkl_advantage_p95": 84.52170700232188, + "kd/rkl_advantage_std": 33.96986777087053, + "kd/ssd_loss": 0.0, + "learning_rate": 2.6635780199749366e-07, + "loss": -13.140496826171875, + "num_tokens": 224974260.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0457352836926779, + "sampling/importance_sampling_ratio/mean": 1.0000193198521932, + "sampling/importance_sampling_ratio/min": 0.9533472716808319, + "sampling/sampling_logp_difference/max": 0.07984772977263978, + "sampling/sampling_logp_difference/mean": 0.001311673268598194, + "step": 19320, + "step_time": 4.259549339265019, + "student/entropy": 0.04955050641049941 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.866666666666666, + "completions/max_terminated_length": 5.866666666666666, + "completions/mean_length": 1.0644444664319357, + "completions/mean_terminated_length": 1.0644444664319357, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05314322377865513, + "epoch": 0.7348194280940266, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.03186321258545, + "kd/policy_loss": -2.0484983603159588, + "kd/rkl_advantage_mean": 46.08451555569967, + "kd/rkl_advantage_p95": 62.26030465761821, + "kd/rkl_advantage_std": 26.335616570711135, + "kd/ssd_loss": 0.0, + "learning_rate": 2.652185470702161e-07, + "loss": -8.193993123372396, + "num_tokens": 225296460.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0278347611427308, + "sampling/importance_sampling_ratio/mean": 0.999950091044108, + "sampling/importance_sampling_ratio/min": 0.9650614023208618, + "sampling/sampling_logp_difference/max": 0.05099525003073116, + "sampling/sampling_logp_difference/mean": 0.001131258915605334, + "step": 19350, + "step_time": 4.273430094167512, + "student/entropy": 0.05314322377865513 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.2, + "completions/max_terminated_length": 9.2, + "completions/mean_length": 1.0963889280954997, + "completions/mean_terminated_length": 1.0963889280954997, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05303617700313528, + "epoch": 0.7359586830213041, + "eval/gt_acc_batch": 0.002222222338120143, + "frac_reward_zero_std": 1.0, + "grad_norm": 34.26569366455078, + "kd/policy_loss": -5.0319268822669985, + "kd/rkl_advantage_mean": 86.35791217486063, + "kd/rkl_advantage_p95": 112.51646482149759, + "kd/rkl_advantage_std": 44.603410822153094, + "kd/ssd_loss": 0.0, + "learning_rate": 2.640792921429385e-07, + "loss": -20.127705891927082, + "num_tokens": 225599375.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.002222222338120143, + "rewards/gt_logging_reward/std": 0.022542615234851838, + "sampling/importance_sampling_ratio/max": 1.024918222427368, + "sampling/importance_sampling_ratio/mean": 0.9994539042313894, + "sampling/importance_sampling_ratio/min": 0.9267344335714977, + "sampling/sampling_logp_difference/max": 0.10181569224223494, + "sampling/sampling_logp_difference/mean": 0.0015543174881410475, + "step": 19380, + "step_time": 4.155797450027603, + "student/entropy": 0.05303617700313528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0461111307144164, + "completions/mean_terminated_length": 1.0461111307144164, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.042849018207440775, + "epoch": 0.7370979379485816, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 34.61155700683594, + "kd/policy_loss": -0.871630831559499, + "kd/rkl_advantage_mean": 45.0804060459137, + "kd/rkl_advantage_p95": 62.07377819220225, + "kd/rkl_advantage_std": 28.375803647438683, + "kd/ssd_loss": 0.0, + "learning_rate": 2.629400372156609e-07, + "loss": -3.486523183186849, + "num_tokens": 225896829.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0271822611490886, + "sampling/importance_sampling_ratio/mean": 0.9998897810777029, + "sampling/importance_sampling_ratio/min": 0.9617450416088105, + "sampling/sampling_logp_difference/max": 0.06420677516919872, + "sampling/sampling_logp_difference/mean": 0.0009553408255063308, + "step": 19410, + "step_time": 4.153451234368549, + "student/entropy": 0.042849018207440775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.033333333333333, + "completions/max_terminated_length": 6.033333333333333, + "completions/mean_length": 1.0600000222524006, + "completions/mean_terminated_length": 1.0600000222524006, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04491606227432688, + "epoch": 0.7382371928758592, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.132404327392578, + "kd/policy_loss": -1.4862577239672343, + "kd/rkl_advantage_mean": 42.58347225189209, + "kd/rkl_advantage_p95": 56.33205201625824, + "kd/rkl_advantage_std": 23.87122474809488, + "kd/ssd_loss": 0.0, + "learning_rate": 2.618007822883834e-07, + "loss": -5.945030721028646, + "num_tokens": 226204829.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0804530501365661, + "sampling/importance_sampling_ratio/mean": 1.000397620598475, + "sampling/importance_sampling_ratio/min": 0.9654175917307536, + "sampling/sampling_logp_difference/max": 0.08576785993451873, + "sampling/sampling_logp_difference/mean": 0.0011415281895703325, + "step": 19440, + "step_time": 4.224096509102674, + "student/entropy": 0.04491606227432688 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.2, + "completions/max_terminated_length": 6.2, + "completions/mean_length": 1.0575000246365864, + "completions/mean_terminated_length": 1.0575000246365864, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05182779027769963, + "epoch": 0.7393764478031367, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 70.38421630859375, + "kd/policy_loss": -1.4485716342926025, + "kd/rkl_advantage_mean": 46.103650856018064, + "kd/rkl_advantage_p95": 62.28917980194092, + "kd/rkl_advantage_std": 26.993567208449047, + "kd/ssd_loss": 0.0, + "learning_rate": 2.606615273611058e-07, + "loss": -5.7942860921223955, + "num_tokens": 226513020.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.05067640542984, + "sampling/importance_sampling_ratio/mean": 1.0000305195649466, + "sampling/importance_sampling_ratio/min": 0.9563450396060944, + "sampling/sampling_logp_difference/max": 0.08403859846293926, + "sampling/sampling_logp_difference/mean": 0.001305623536851878, + "step": 19470, + "step_time": 4.23079491082996, + "student/entropy": 0.05182779027769963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.3, + "completions/max_terminated_length": 7.3, + "completions/mean_length": 1.0800000309944153, + "completions/mean_terminated_length": 1.0800000309944153, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0631954128233095, + "epoch": 0.7405157027304143, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 41.2197265625, + "kd/policy_loss": -3.277601762612661, + "kd/rkl_advantage_mean": 57.202845366795856, + "kd/rkl_advantage_p95": 73.51731602350871, + "kd/rkl_advantage_std": 28.595030107100804, + "kd/ssd_loss": 0.0, + "learning_rate": 2.595222724338282e-07, + "loss": -13.110407511393229, + "num_tokens": 226826572.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.038950781027476, + "sampling/importance_sampling_ratio/mean": 0.9998475929101308, + "sampling/importance_sampling_ratio/min": 0.9529669443766277, + "sampling/sampling_logp_difference/max": 0.1167227401242902, + "sampling/sampling_logp_difference/mean": 0.0016276635346002878, + "step": 19500, + "step_time": 4.231827859565965, + "student/entropy": 0.0631954128233095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.766666666666667, + "completions/max_terminated_length": 6.766666666666667, + "completions/mean_length": 1.0577778021494548, + "completions/mean_terminated_length": 1.0577778021494548, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05562592806915442, + "epoch": 0.7416549576576918, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.603557586669922, + "kd/policy_loss": -1.6167506297429404, + "kd/rkl_advantage_mean": 57.27709908485413, + "kd/rkl_advantage_p95": 78.62033898830414, + "kd/rkl_advantage_std": 35.39020743568738, + "kd/ssd_loss": 0.0, + "learning_rate": 2.5838301750655074e-07, + "loss": -6.467002360026042, + "num_tokens": 227122172.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0061082363128662, + "sampling/importance_sampling_ratio/mean": 0.9996979475021363, + "sampling/importance_sampling_ratio/min": 0.9624804496765137, + "sampling/sampling_logp_difference/max": 0.03774979105219245, + "sampling/sampling_logp_difference/mean": 0.000859555511851795, + "step": 19530, + "step_time": 4.205847303802147, + "student/entropy": 0.05562592806915442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.466666666666667, + "completions/max_terminated_length": 6.466666666666667, + "completions/mean_length": 1.0663889129956563, + "completions/mean_terminated_length": 1.0663889129956563, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052821204500893754, + "epoch": 0.7427942125849695, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.349191665649414, + "kd/policy_loss": -1.8471262256304424, + "kd/rkl_advantage_mean": 50.80897208849589, + "kd/rkl_advantage_p95": 67.48871766726175, + "kd/rkl_advantage_std": 28.790036537249883, + "kd/ssd_loss": 0.0, + "learning_rate": 2.5724376257927316e-07, + "loss": -7.3885040283203125, + "num_tokens": 227425043.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0063138961791993, + "sampling/importance_sampling_ratio/mean": 0.9997319142023723, + "sampling/importance_sampling_ratio/min": 0.9638502717018127, + "sampling/sampling_logp_difference/max": 0.0396559817250818, + "sampling/sampling_logp_difference/mean": 0.0007979621276414643, + "step": 19560, + "step_time": 4.197667686700394, + "student/entropy": 0.052821204500893754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.533333333333333, + "completions/max_terminated_length": 5.533333333333333, + "completions/mean_length": 1.0608333547910054, + "completions/mean_terminated_length": 1.0608333547910054, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06138023342937231, + "epoch": 0.743933467512247, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 57.29224395751953, + "kd/policy_loss": -0.7466307322184245, + "kd/rkl_advantage_mean": 37.02867914835612, + "kd/rkl_advantage_p95": 50.551554274559024, + "kd/rkl_advantage_std": 22.721012363831203, + "kd/ssd_loss": 0.0, + "learning_rate": 2.561045076519956e-07, + "loss": -2.986522165934245, + "num_tokens": 227741822.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0256899833679198, + "sampling/importance_sampling_ratio/mean": 0.999905397494634, + "sampling/importance_sampling_ratio/min": 0.9683597485224406, + "sampling/sampling_logp_difference/max": 0.047390998201444746, + "sampling/sampling_logp_difference/mean": 0.001038375041874436, + "step": 19590, + "step_time": 4.195509042234335, + "student/entropy": 0.06138023342937231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0005555555845300357, + "completions/max_length": 41.56666666666667, + "completions/max_terminated_length": 7.9, + "completions/mean_length": 1.4102778315544129, + "completions/mean_terminated_length": 1.126468288898468, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06851071082055568, + "epoch": 0.7450727224395246, + "eval/gt_acc_batch": 0.003333333507180214, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.836156845092773, + "kd/policy_loss": -5.36651861568292, + "kd/rkl_advantage_mean": 74.43386456171672, + "kd/rkl_advantage_p95": 90.5340839544932, + "kd/rkl_advantage_std": 29.21109362343947, + "kd/ssd_loss": 0.0, + "learning_rate": 2.5496525272471805e-07, + "loss": -21.466070556640624, + "num_tokens": 228054795.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.003333333507180214, + "rewards/gt_logging_reward/std": 0.03111300617456436, + "sampling/importance_sampling_ratio/max": 1.0732258001963297, + "sampling/importance_sampling_ratio/mean": 1.0000862896442413, + "sampling/importance_sampling_ratio/min": 0.9395048578580221, + "sampling/sampling_logp_difference/max": 0.08760479263340433, + "sampling/sampling_logp_difference/mean": 0.0017943147366167977, + "step": 19620, + "step_time": 4.358565194265975, + "student/entropy": 0.06851071082055568 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.766666666666667, + "completions/max_terminated_length": 6.766666666666667, + "completions/mean_length": 1.0700000286102296, + "completions/mean_terminated_length": 1.0700000286102296, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06234233311067025, + "epoch": 0.7462119773668021, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 53.29075622558594, + "kd/policy_loss": -2.5270787835121156, + "kd/rkl_advantage_mean": 54.49156198501587, + "kd/rkl_advantage_p95": 73.17553160985311, + "kd/rkl_advantage_std": 30.23896225988865, + "kd/ssd_loss": 0.0, + "learning_rate": 2.5382599779744047e-07, + "loss": -10.108316040039062, + "num_tokens": 228356279.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0254994312922159, + "sampling/importance_sampling_ratio/mean": 0.9998343745867412, + "sampling/importance_sampling_ratio/min": 0.9515795330206553, + "sampling/sampling_logp_difference/max": 0.056937929879253106, + "sampling/sampling_logp_difference/mean": 0.0011489000549772753, + "step": 19650, + "step_time": 4.083927451563553, + "student/entropy": 0.06234233311067025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.066666666666666, + "completions/max_terminated_length": 8.066666666666666, + "completions/mean_length": 1.0847222487131754, + "completions/mean_terminated_length": 1.0847222487131754, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.058950991928577424, + "epoch": 0.7473512322940796, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.207494735717773, + "kd/policy_loss": -4.032014063994089, + "kd/rkl_advantage_mean": 70.61388231913249, + "kd/rkl_advantage_p95": 90.73081905047098, + "kd/rkl_advantage_std": 35.64014074504375, + "kd/ssd_loss": 0.0, + "learning_rate": 2.526867428701629e-07, + "loss": -16.128054809570312, + "num_tokens": 228668176.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0307003299395243, + "sampling/importance_sampling_ratio/mean": 0.9997425377368927, + "sampling/importance_sampling_ratio/min": 0.941946283976237, + "sampling/sampling_logp_difference/max": 0.08353097671642899, + "sampling/sampling_logp_difference/mean": 0.0013561092016364758, + "step": 19680, + "step_time": 4.217069577168634, + "student/entropy": 0.058950991928577424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.8, + "completions/max_terminated_length": 7.8, + "completions/mean_length": 1.0675000230471292, + "completions/mean_terminated_length": 1.0675000230471292, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05356613928452134, + "epoch": 0.7484904872213572, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.37274742126465, + "kd/policy_loss": -1.9877112269401551, + "kd/rkl_advantage_mean": 59.626633961995445, + "kd/rkl_advantage_p95": 78.92128971417745, + "kd/rkl_advantage_std": 33.382076829175155, + "kd/ssd_loss": 0.0, + "learning_rate": 2.515474879428853e-07, + "loss": -7.95084228515625, + "num_tokens": 228977467.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.095082974433899, + "sampling/importance_sampling_ratio/mean": 1.0006135523319244, + "sampling/importance_sampling_ratio/min": 0.9765601674715678, + "sampling/sampling_logp_difference/max": 0.07973543028347194, + "sampling/sampling_logp_difference/mean": 0.0012628253820973138, + "step": 19710, + "step_time": 4.191670813435242, + "student/entropy": 0.05356613928452134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.433333333333334, + "completions/max_terminated_length": 7.433333333333334, + "completions/mean_length": 1.0627778053283692, + "completions/mean_terminated_length": 1.0627778053283692, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05377276924749216, + "epoch": 0.7496297421486348, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.574056625366211, + "kd/policy_loss": -2.9852620164553323, + "kd/rkl_advantage_mean": 63.83854471842448, + "kd/rkl_advantage_p95": 84.49895305633545, + "kd/rkl_advantage_std": 35.24234878023466, + "kd/ssd_loss": 0.0, + "learning_rate": 2.5040823301560777e-07, + "loss": -11.941048177083333, + "num_tokens": 229269557.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0263959010442099, + "sampling/importance_sampling_ratio/mean": 0.9998697181542714, + "sampling/importance_sampling_ratio/min": 0.9597635527451833, + "sampling/sampling_logp_difference/max": 0.05964795281179249, + "sampling/sampling_logp_difference/mean": 0.0010203697165707126, + "step": 19740, + "step_time": 4.170766320396312, + "student/entropy": 0.05377276924749216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.5, + "completions/max_terminated_length": 6.5, + "completions/mean_length": 1.067222245534261, + "completions/mean_terminated_length": 1.067222245534261, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0485066132619977, + "epoch": 0.7507689970759124, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.867323875427246, + "kd/policy_loss": -2.14902370373408, + "kd/rkl_advantage_mean": 51.186043961842856, + "kd/rkl_advantage_p95": 67.69474090735117, + "kd/rkl_advantage_std": 28.275479504962764, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4926897808833024e-07, + "loss": -8.59609375, + "num_tokens": 229578951.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0258677879969278, + "sampling/importance_sampling_ratio/mean": 0.9997812827428182, + "sampling/importance_sampling_ratio/min": 0.9501366376876831, + "sampling/sampling_logp_difference/max": 0.06555539335434636, + "sampling/sampling_logp_difference/mean": 0.0010302396869519726, + "step": 19770, + "step_time": 4.225739528268362, + "student/entropy": 0.0485066132619977 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 22.133333333333333, + "completions/max_terminated_length": 5.1, + "completions/mean_length": 1.1988889217376708, + "completions/mean_terminated_length": 1.0569444656372071, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05531858603159587, + "epoch": 0.7519082520031899, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.683964729309082, + "kd/policy_loss": -0.5492961496114731, + "kd/rkl_advantage_mean": 37.49390012423198, + "kd/rkl_advantage_p95": 50.76525913874308, + "kd/rkl_advantage_std": 21.54698723355929, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4812972316105266e-07, + "loss": -2.197184753417969, + "num_tokens": 229892195.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.058070953687032, + "sampling/importance_sampling_ratio/mean": 1.0001801252365112, + "sampling/importance_sampling_ratio/min": 0.9697714805603027, + "sampling/sampling_logp_difference/max": 0.07327667507342994, + "sampling/sampling_logp_difference/mean": 0.0011722612058899056, + "step": 19800, + "step_time": 4.261271480369517, + "student/entropy": 0.05531858603159587 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.833333333333333, + "completions/max_terminated_length": 7.833333333333333, + "completions/mean_length": 1.0905555923779806, + "completions/mean_terminated_length": 1.0905555923779806, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06655161368350188, + "epoch": 0.7530475069304675, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 51.486289978027344, + "kd/policy_loss": -5.565947870413463, + "kd/rkl_advantage_mean": 71.81413515408833, + "kd/rkl_advantage_p95": 90.65857542355856, + "kd/rkl_advantage_std": 32.404118042190866, + "kd/ssd_loss": 0.0, + "learning_rate": 2.469904682337751e-07, + "loss": -22.263787841796876, + "num_tokens": 230191601.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.019499711940685907, + "sampling/importance_sampling_ratio/max": 1.0410502552986145, + "sampling/importance_sampling_ratio/mean": 0.9997103333473205, + "sampling/importance_sampling_ratio/min": 0.9343806227048238, + "sampling/sampling_logp_difference/max": 0.11028483527091643, + "sampling/sampling_logp_difference/mean": 0.0017346894252113997, + "step": 19830, + "step_time": 4.242579053199734, + "student/entropy": 0.06655161368350188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.533333333333333, + "completions/max_terminated_length": 7.533333333333333, + "completions/mean_length": 1.0858333706855774, + "completions/mean_terminated_length": 1.0858333706855774, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05562555758903424, + "epoch": 0.754186761857745, + "eval/gt_acc_batch": 0.002222222338120143, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.091314315795898, + "kd/policy_loss": -4.053225662310918, + "kd/rkl_advantage_mean": 68.56968646049499, + "kd/rkl_advantage_p95": 90.8514505704244, + "kd/rkl_advantage_std": 36.102969686190285, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4585121330649755e-07, + "loss": -16.212901814778647, + "num_tokens": 230507294.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.002222222338120143, + "rewards/gt_logging_reward/std": 0.022542615234851838, + "sampling/importance_sampling_ratio/max": 1.0571131507555644, + "sampling/importance_sampling_ratio/mean": 1.0001038988431294, + "sampling/importance_sampling_ratio/min": 0.9468689302603404, + "sampling/sampling_logp_difference/max": 0.09632109257703027, + "sampling/sampling_logp_difference/mean": 0.0015036877380528797, + "step": 19860, + "step_time": 4.268945549497342, + "student/entropy": 0.05562555758903424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.666666666666667, + "completions/max_terminated_length": 5.666666666666667, + "completions/mean_length": 1.0458333571751912, + "completions/mean_terminated_length": 1.0458333571751912, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05108618251979351, + "epoch": 0.7553260167850226, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.54359245300293, + "kd/policy_loss": -1.1609881401062012, + "kd/rkl_advantage_mean": 45.12697114944458, + "kd/rkl_advantage_p95": 67.48947933514913, + "kd/rkl_advantage_std": 31.07405738333861, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4471195837921997e-07, + "loss": -4.643953450520834, + "num_tokens": 230822699.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0434563159942627, + "sampling/importance_sampling_ratio/mean": 0.9999646921952565, + "sampling/importance_sampling_ratio/min": 0.9580431858698527, + "sampling/sampling_logp_difference/max": 0.06836722881222765, + "sampling/sampling_logp_difference/mean": 0.0011296491468480478, + "step": 19890, + "step_time": 4.217374576633059, + "student/entropy": 0.05108618251979351 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.233333333333333, + "completions/max_terminated_length": 6.233333333333333, + "completions/mean_length": 1.0550000230471293, + "completions/mean_terminated_length": 1.0550000230471293, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05354052536810438, + "epoch": 0.7564652717123002, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.955324172973633, + "kd/policy_loss": -1.552400243282318, + "kd/rkl_advantage_mean": 49.81686970392863, + "kd/rkl_advantage_p95": 67.388503074646, + "kd/rkl_advantage_std": 29.736733762423196, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4357270345194244e-07, + "loss": -6.209602355957031, + "num_tokens": 231127905.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0246466755867005, + "sampling/importance_sampling_ratio/mean": 0.999855907758077, + "sampling/importance_sampling_ratio/min": 0.9577935397624969, + "sampling/sampling_logp_difference/max": 0.051334485341794786, + "sampling/sampling_logp_difference/mean": 0.0008991590448810409, + "step": 19920, + "step_time": 4.185202891501346, + "student/entropy": 0.05354052536810438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.3, + "completions/max_terminated_length": 8.3, + "completions/mean_length": 1.0883333722750346, + "completions/mean_terminated_length": 1.0883333722750346, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.053039074099312224, + "epoch": 0.7576045266395777, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.552802085876465, + "kd/policy_loss": -4.4282717188199365, + "kd/rkl_advantage_mean": 74.27597317695617, + "kd/rkl_advantage_p95": 96.10243926843008, + "kd/rkl_advantage_std": 37.37151495317618, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4243344852466486e-07, + "loss": -17.7130859375, + "num_tokens": 231426863.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0471065799395243, + "sampling/importance_sampling_ratio/mean": 0.9996942659219106, + "sampling/importance_sampling_ratio/min": 0.9350918849309285, + "sampling/sampling_logp_difference/max": 0.11297684788393478, + "sampling/sampling_logp_difference/mean": 0.001612784446721586, + "step": 19950, + "step_time": 4.199756263866827, + "student/entropy": 0.053039074099312224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 24.766666666666666, + "completions/max_terminated_length": 8.0, + "completions/mean_length": 1.2236111521720887, + "completions/mean_terminated_length": 1.0816853721936543, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.046283973443011446, + "epoch": 0.7587437815668553, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.155189514160156, + "kd/policy_loss": -3.733914369344711, + "kd/rkl_advantage_mean": 74.29688811302185, + "kd/rkl_advantage_p95": 96.06314513683319, + "kd/rkl_advantage_std": 37.561005448301636, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4129419359738727e-07, + "loss": -14.935655721028645, + "num_tokens": 231726172.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0935733159383139, + "sampling/importance_sampling_ratio/mean": 1.0003610094388327, + "sampling/importance_sampling_ratio/min": 0.9548847615718842, + "sampling/sampling_logp_difference/max": 0.0904938921953241, + "sampling/sampling_logp_difference/mean": 0.0010674304595643966, + "step": 19980, + "step_time": 4.262599625233755, + "student/entropy": 0.046283973443011446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 3.2, + "completions/max_terminated_length": 3.2, + "completions/mean_length": 1.0244444568951925, + "completions/mean_terminated_length": 1.0244444568951925, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04866629764437676, + "epoch": 0.7598830364941328, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.156922340393066, + "kd/policy_loss": 2.016217581431071, + "kd/rkl_advantage_mean": 10.351472965876262, + "kd/rkl_advantage_p95": 22.37694275379181, + "kd/rkl_advantage_std": 15.093949186801911, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4015493867010974e-07, + "loss": 8.064871215820313, + "num_tokens": 232029324.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0318130691846212, + "sampling/importance_sampling_ratio/mean": 1.000047395626704, + "sampling/importance_sampling_ratio/min": 0.9731359263261159, + "sampling/sampling_logp_difference/max": 0.05205971458150695, + "sampling/sampling_logp_difference/mean": 0.0009134926217181298, + "step": 20010, + "step_time": 4.2245715046015295, + "student/entropy": 0.04866629764437676 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.6, + "completions/max_terminated_length": 7.6, + "completions/mean_length": 1.097777815659841, + "completions/mean_terminated_length": 1.097777815659841, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06652997586255273, + "epoch": 0.7610222914214104, + "eval/gt_acc_batch": 0.0025000001303851606, + "frac_reward_zero_std": 1.0, + "grad_norm": 23.511686325073242, + "kd/policy_loss": -5.236852280298868, + "kd/rkl_advantage_mean": 63.29381165504456, + "kd/rkl_advantage_p95": 79.30795207023621, + "kd/rkl_advantage_std": 27.183113782604536, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3901568374283216e-07, + "loss": -20.947408040364582, + "num_tokens": 232343676.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000001303851606, + "rewards/gt_logging_reward/std": 0.02378490741054217, + "sampling/importance_sampling_ratio/max": 1.090949006875356, + "sampling/importance_sampling_ratio/mean": 1.0006875077883401, + "sampling/importance_sampling_ratio/min": 0.9681107223033905, + "sampling/sampling_logp_difference/max": 0.08409845766921838, + "sampling/sampling_logp_difference/mean": 0.0015534348834383611, + "step": 20040, + "step_time": 4.277249319831996, + "student/entropy": 0.06652997586255273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.666666666666667, + "completions/max_terminated_length": 7.666666666666667, + "completions/mean_length": 1.0802778085072835, + "completions/mean_terminated_length": 1.0802778085072835, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06064321997885903, + "epoch": 0.7621615463486879, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.784564971923828, + "kd/policy_loss": -3.9524887641270956, + "kd/rkl_advantage_mean": 65.85368140538533, + "kd/rkl_advantage_p95": 84.59130074977875, + "kd/rkl_advantage_std": 32.671480265259746, + "kd/ssd_loss": 0.0, + "learning_rate": 2.378764288155546e-07, + "loss": -15.80995585123698, + "num_tokens": 232652309.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0492122928301493, + "sampling/importance_sampling_ratio/mean": 0.9998740196228028, + "sampling/importance_sampling_ratio/min": 0.9343250513076782, + "sampling/sampling_logp_difference/max": 0.07966345966172715, + "sampling/sampling_logp_difference/mean": 0.0015009191498393192, + "step": 20070, + "step_time": 4.247736018567229, + "student/entropy": 0.06064321997885903 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 25.066666666666666, + "completions/max_terminated_length": 8.033333333333333, + "completions/mean_length": 1.2302778204282125, + "completions/mean_terminated_length": 1.0883333643277486, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05123585623999437, + "epoch": 0.7633008012759656, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.77803611755371, + "kd/policy_loss": -4.1736560742060345, + "kd/rkl_advantage_mean": 75.52362171808879, + "kd/rkl_advantage_p95": 96.30723976294199, + "kd/rkl_advantage_std": 36.32731396754583, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3673717388827705e-07, + "loss": -16.694624837239584, + "num_tokens": 232966434.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0572971820831298, + "sampling/importance_sampling_ratio/mean": 1.0000592033068338, + "sampling/importance_sampling_ratio/min": 0.9589306672414144, + "sampling/sampling_logp_difference/max": 0.09067870667204261, + "sampling/sampling_logp_difference/mean": 0.001185051777671712, + "step": 20100, + "step_time": 4.286919491800655, + "student/entropy": 0.05123585623999437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0450000206629435, + "completions/mean_terminated_length": 1.0450000206629435, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04630657993257046, + "epoch": 0.7644400562032431, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.622546195983887, + "kd/policy_loss": -0.8929827253023783, + "kd/rkl_advantage_mean": 51.94691410064697, + "kd/rkl_advantage_p95": 73.02637255986532, + "kd/rkl_advantage_std": 33.97980794012547, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3559791896099952e-07, + "loss": -3.5719309488932294, + "num_tokens": 233278332.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0724622448285421, + "sampling/importance_sampling_ratio/mean": 1.0005578180154164, + "sampling/importance_sampling_ratio/min": 0.9894974013169606, + "sampling/sampling_logp_difference/max": 0.059502584487199785, + "sampling/sampling_logp_difference/mean": 0.0009586189708594854, + "step": 20130, + "step_time": 4.22168326320049, + "student/entropy": 0.04630657993257046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.827586206896552, + "completions/max_terminated_length": 6.827586206896552, + "completions/mean_length": 1.070402326255009, + "completions/mean_terminated_length": 1.070402326255009, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04914234649261524, + "epoch": 0.7655793111305207, + "eval/gt_acc_batch": 0.0017241380209552831, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.735086441040039, + "kd/policy_loss": -2.040087342262268, + "kd/rkl_advantage_mean": 51.682411358274265, + "kd/rkl_advantage_p95": 70.5331520294321, + "kd/rkl_advantage_std": 29.851567959990994, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3445866403372194e-07, + "loss": -7.888338216145834, + "num_tokens": 233570081.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0017241380209552831, + "rewards/gt_logging_reward/std": 0.017024284806744807, + "sampling/importance_sampling_ratio/max": 1.0455900266252716, + "sampling/importance_sampling_ratio/mean": 1.00001132693784, + "sampling/importance_sampling_ratio/min": 0.9587818281403904, + "sampling/sampling_logp_difference/max": 0.07958374933564458, + "sampling/sampling_logp_difference/mean": 0.001251782000602769, + "step": 20160, + "step_time": 4.08613935213604, + "student/entropy": 0.04914234649261524 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.0480555733044943, + "completions/mean_terminated_length": 1.0480555733044943, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04948433491711815, + "epoch": 0.7667185660577982, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 39.94668960571289, + "kd/policy_loss": -0.5438123305638631, + "kd/rkl_advantage_mean": 44.34812927246094, + "kd/rkl_advantage_p95": 61.94325869878133, + "kd/rkl_advantage_std": 28.93360018928846, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3331940910644438e-07, + "loss": -2.175248718261719, + "num_tokens": 233870350.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.058479412396749, + "sampling/importance_sampling_ratio/mean": 1.000298986832301, + "sampling/importance_sampling_ratio/min": 0.9757758537928264, + "sampling/sampling_logp_difference/max": 0.06773189761055012, + "sampling/sampling_logp_difference/mean": 0.000989950165482393, + "step": 20190, + "step_time": 4.189365102000011, + "student/entropy": 0.04948433491711815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.366666666666667, + "completions/max_terminated_length": 8.366666666666667, + "completions/mean_length": 1.086388921737671, + "completions/mean_terminated_length": 1.086388921737671, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05393142520139615, + "epoch": 0.7678578209850757, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.707819938659668, + "kd/policy_loss": -4.527824457486471, + "kd/rkl_advantage_mean": 75.30814638137818, + "kd/rkl_advantage_p95": 96.44411067962646, + "kd/rkl_advantage_std": 37.42760017116864, + "kd/ssd_loss": 0.0, + "learning_rate": 2.321801541791668e-07, + "loss": -18.111299641927083, + "num_tokens": 234173733.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.047029980023702, + "sampling/importance_sampling_ratio/mean": 0.9999754309654236, + "sampling/importance_sampling_ratio/min": 0.9409174144268035, + "sampling/sampling_logp_difference/max": 0.07864265370493134, + "sampling/sampling_logp_difference/mean": 0.001452855843429764, + "step": 20220, + "step_time": 4.1723479855364225, + "student/entropy": 0.05393142520139615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.6, + "completions/max_terminated_length": 6.6, + "completions/mean_length": 1.0600000222524006, + "completions/mean_terminated_length": 1.0600000222524006, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04854706563055515, + "epoch": 0.7689970759123533, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.934139251708984, + "kd/policy_loss": -1.6196706573168436, + "kd/rkl_advantage_mean": 50.99187343915303, + "kd/rkl_advantage_p95": 67.90932459036509, + "kd/rkl_advantage_std": 29.47297716041406, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3104089925188924e-07, + "loss": -6.4786834716796875, + "num_tokens": 234484005.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0202759186426797, + "sampling/importance_sampling_ratio/mean": 0.9997497141361237, + "sampling/importance_sampling_ratio/min": 0.9564437508583069, + "sampling/sampling_logp_difference/max": 0.059004688278461496, + "sampling/sampling_logp_difference/mean": 0.0009171163149100418, + "step": 20250, + "step_time": 4.324163994836272, + "student/entropy": 0.04854706563055515 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.433333333333334, + "completions/max_terminated_length": 6.433333333333334, + "completions/mean_length": 1.0536111315091452, + "completions/mean_terminated_length": 1.0536111315091452, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04911769954487681, + "epoch": 0.7701363308396308, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 61.20832443237305, + "kd/policy_loss": -1.1110673069953918, + "kd/rkl_advantage_mean": 45.267688512802124, + "kd/rkl_advantage_p95": 62.07918462753296, + "kd/rkl_advantage_std": 27.8172111560901, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2990164432461171e-07, + "loss": -4.4442693074544275, + "num_tokens": 234790206.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0191657781600951, + "sampling/importance_sampling_ratio/mean": 0.99987646540006, + "sampling/importance_sampling_ratio/min": 0.971196695168813, + "sampling/sampling_logp_difference/max": 0.05447421919864913, + "sampling/sampling_logp_difference/mean": 0.0008958819550268041, + "step": 20280, + "step_time": 4.243590718467021, + "student/entropy": 0.04911769954487681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.566666666666666, + "completions/max_terminated_length": 5.566666666666666, + "completions/mean_length": 1.0561111330986024, + "completions/mean_terminated_length": 1.0561111330986024, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05453920643776655, + "epoch": 0.7712755857669085, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 39.58537673950195, + "kd/policy_loss": -1.1143791794776916, + "kd/rkl_advantage_mean": 41.42128721872965, + "kd/rkl_advantage_p95": 56.60953923861186, + "kd/rkl_advantage_std": 25.150244814157485, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2876238939733413e-07, + "loss": -4.457515462239583, + "num_tokens": 235101440.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0148091793060303, + "sampling/importance_sampling_ratio/mean": 0.9999457418918609, + "sampling/importance_sampling_ratio/min": 0.9736619710922241, + "sampling/sampling_logp_difference/max": 0.04926017403292159, + "sampling/sampling_logp_difference/mean": 0.0009375890222145244, + "step": 20310, + "step_time": 4.248757150802703, + "student/entropy": 0.05453920643776655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.1, + "completions/max_terminated_length": 9.1, + "completions/mean_length": 1.1133333722750345, + "completions/mean_terminated_length": 1.1133333722750345, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05821894767383735, + "epoch": 0.772414840694186, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.340036392211914, + "kd/policy_loss": -6.052039794127146, + "kd/rkl_advantage_mean": 84.86338650385538, + "kd/rkl_advantage_p95": 107.52220607598623, + "kd/rkl_advantage_std": 39.61987656652927, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2762313447005658e-07, + "loss": -24.208158365885417, + "num_tokens": 235409464.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0249979813893637, + "sampling/importance_sampling_ratio/mean": 0.9996805389722189, + "sampling/importance_sampling_ratio/min": 0.94870645403862, + "sampling/sampling_logp_difference/max": 0.07574834050610661, + "sampling/sampling_logp_difference/mean": 0.0012413677322911099, + "step": 20340, + "step_time": 4.212752504598757, + "student/entropy": 0.05821894767383735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.633333333333334, + "completions/max_terminated_length": 4.633333333333334, + "completions/mean_length": 1.05444446404775, + "completions/mean_terminated_length": 1.05444446404775, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05558656721065442, + "epoch": 0.7735540956214636, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.350122451782227, + "kd/policy_loss": -1.153570787111918, + "kd/rkl_advantage_mean": 29.10137832959493, + "kd/rkl_advantage_p95": 39.298574368158974, + "kd/rkl_advantage_std": 16.271720245977242, + "kd/ssd_loss": 0.0, + "learning_rate": 2.26483879542779e-07, + "loss": -4.614283752441406, + "num_tokens": 235716836.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.023202900091807, + "sampling/importance_sampling_ratio/mean": 0.9999887843926748, + "sampling/importance_sampling_ratio/min": 0.969200599193573, + "sampling/sampling_logp_difference/max": 0.03153188788952927, + "sampling/sampling_logp_difference/mean": 0.0008862353890435771, + "step": 20370, + "step_time": 4.236413901501025, + "student/entropy": 0.05558656721065442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 24.666666666666668, + "completions/max_terminated_length": 7.633333333333334, + "completions/mean_length": 1.2233333746592203, + "completions/mean_terminated_length": 1.0813889185587564, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.059733786682287855, + "epoch": 0.7746933505487411, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.417801856994629, + "kd/policy_loss": -3.9237802396217982, + "kd/rkl_advantage_mean": 71.16208572387696, + "kd/rkl_advantage_p95": 90.1112292766571, + "kd/rkl_advantage_std": 33.97065525650978, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2534462461550144e-07, + "loss": -15.695121256510417, + "num_tokens": 236020080.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0682285706202188, + "sampling/importance_sampling_ratio/mean": 1.0000772178173065, + "sampling/importance_sampling_ratio/min": 0.942377895116806, + "sampling/sampling_logp_difference/max": 0.10141176106408238, + "sampling/sampling_logp_difference/mean": 0.0014152066597792631, + "step": 20400, + "step_time": 4.212740567600122, + "student/entropy": 0.059733786682287855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.0688889145851135, + "completions/mean_terminated_length": 1.0688889145851135, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05283599483470122, + "epoch": 0.7758326054760187, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.333199501037598, + "kd/policy_loss": -2.331309374173482, + "kd/rkl_advantage_mean": 55.70885019302368, + "kd/rkl_advantage_p95": 73.51643419265747, + "kd/rkl_advantage_std": 30.68362290263176, + "kd/ssd_loss": 0.0, + "learning_rate": 2.242053696882239e-07, + "loss": -9.325240071614584, + "num_tokens": 236327376.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0238639156023661, + "sampling/importance_sampling_ratio/mean": 0.9998364249865214, + "sampling/importance_sampling_ratio/min": 0.9593366185824076, + "sampling/sampling_logp_difference/max": 0.07151311873458326, + "sampling/sampling_logp_difference/mean": 0.0012321533644959952, + "step": 20430, + "step_time": 4.153149602867294, + "student/entropy": 0.05283599483470122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.366666666666666, + "completions/max_terminated_length": 6.366666666666666, + "completions/mean_length": 1.0555555780728658, + "completions/mean_terminated_length": 1.0555555780728658, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05749573176726699, + "epoch": 0.7769718604032962, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.753364562988281, + "kd/policy_loss": -1.2692359964052835, + "kd/rkl_advantage_mean": 49.87320674260457, + "kd/rkl_advantage_p95": 68.06672387917837, + "kd/rkl_advantage_std": 30.476059265931447, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2306611476094633e-07, + "loss": -5.076943969726562, + "num_tokens": 236640344.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0187443296114604, + "sampling/importance_sampling_ratio/mean": 0.9998400767644247, + "sampling/importance_sampling_ratio/min": 0.9626203894615173, + "sampling/sampling_logp_difference/max": 0.056615079194307326, + "sampling/sampling_logp_difference/mean": 0.001001252896579293, + "step": 20460, + "step_time": 4.23487088969899, + "student/entropy": 0.05749573176726699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.8, + "completions/max_terminated_length": 6.8, + "completions/mean_length": 1.0680555820465087, + "completions/mean_terminated_length": 1.0680555820465087, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0582029001476864, + "epoch": 0.7781111153305738, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.534090042114258, + "kd/policy_loss": -2.907151460647583, + "kd/rkl_advantage_mean": 56.118311961491905, + "kd/rkl_advantage_p95": 73.35518583456675, + "kd/rkl_advantage_std": 29.35823458234469, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2192685983366877e-07, + "loss": -11.628607177734375, + "num_tokens": 236938677.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0216301759084065, + "sampling/importance_sampling_ratio/mean": 0.9999578257401784, + "sampling/importance_sampling_ratio/min": 0.9680274347464244, + "sampling/sampling_logp_difference/max": 0.044732139251815774, + "sampling/sampling_logp_difference/mean": 0.0009976245734530191, + "step": 20490, + "step_time": 4.160687628170126, + "student/entropy": 0.0582029001476864 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.666666666666667, + "completions/max_terminated_length": 7.666666666666667, + "completions/mean_length": 1.093333371480306, + "completions/mean_terminated_length": 1.093333371480306, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.06067819915090998, + "epoch": 0.7792503702578514, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.999859809875488, + "kd/policy_loss": -4.03940935532252, + "kd/rkl_advantage_mean": 69.65061508814493, + "kd/rkl_advantage_p95": 90.42694514592489, + "kd/rkl_advantage_std": 35.68454687098662, + "kd/ssd_loss": 0.0, + "learning_rate": 2.207876049063912e-07, + "loss": -16.15763651529948, + "num_tokens": 237249773.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.019499711940685907, + "sampling/importance_sampling_ratio/max": 1.0593228618303934, + "sampling/importance_sampling_ratio/mean": 1.000255000591278, + "sampling/importance_sampling_ratio/min": 0.9714713950951894, + "sampling/sampling_logp_difference/max": 0.0892020209847639, + "sampling/sampling_logp_difference/mean": 0.0014001377916429193, + "step": 20520, + "step_time": 4.117420771066585, + "student/entropy": 0.06067819915090998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.066666666666666, + "completions/max_terminated_length": 7.066666666666666, + "completions/mean_length": 1.0858333587646485, + "completions/mean_terminated_length": 1.0858333587646485, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05423876037821174, + "epoch": 0.7803896251851289, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.42416763305664, + "kd/policy_loss": -3.496575653553009, + "kd/rkl_advantage_mean": 61.15544322331746, + "kd/rkl_advantage_p95": 79.34939707914988, + "kd/rkl_advantage_std": 31.263560343782107, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1964834997911366e-07, + "loss": -13.986301676432292, + "num_tokens": 237557570.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0244020581245423, + "sampling/importance_sampling_ratio/mean": 0.9996652086575826, + "sampling/importance_sampling_ratio/min": 0.9489113787810007, + "sampling/sampling_logp_difference/max": 0.06358670255479713, + "sampling/sampling_logp_difference/mean": 0.0010681935508425036, + "step": 20550, + "step_time": 4.143645820102635, + "student/entropy": 0.05423876037821174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.2, + "completions/max_terminated_length": 5.2, + "completions/mean_length": 1.0536111354827882, + "completions/mean_terminated_length": 1.0536111354827882, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05110643322890004, + "epoch": 0.7815288801124065, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.752700805664062, + "kd/policy_loss": -0.7228954434394836, + "kd/rkl_advantage_mean": 36.57974497477213, + "kd/rkl_advantage_p95": 50.841304779052734, + "kd/rkl_advantage_std": 23.21110623975595, + "kd/ssd_loss": 0.0, + "learning_rate": 2.185090950518361e-07, + "loss": -2.89158198038737, + "num_tokens": 237879387.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.029635981718699, + "sampling/importance_sampling_ratio/mean": 0.9999234954516093, + "sampling/importance_sampling_ratio/min": 0.9599275807539622, + "sampling/sampling_logp_difference/max": 0.061278829568376145, + "sampling/sampling_logp_difference/mean": 0.0009725029754918069, + "step": 20580, + "step_time": 4.2630358858684, + "student/entropy": 0.05110643322890004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.5, + "completions/max_terminated_length": 8.5, + "completions/mean_length": 1.0813889225323996, + "completions/mean_terminated_length": 1.0813889225323996, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.054577818668137, + "epoch": 0.782668135039684, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.52094268798828, + "kd/policy_loss": -4.544773515065511, + "kd/rkl_advantage_mean": 81.67518246968588, + "kd/rkl_advantage_p95": 107.22696644465128, + "kd/rkl_advantage_std": 43.14768689274788, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1736984012455852e-07, + "loss": -18.179097493489582, + "num_tokens": 238194816.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0630279580752056, + "sampling/importance_sampling_ratio/mean": 1.0000856359799704, + "sampling/importance_sampling_ratio/min": 0.940766034523646, + "sampling/sampling_logp_difference/max": 0.10199598995968699, + "sampling/sampling_logp_difference/mean": 0.001398647814251793, + "step": 20610, + "step_time": 4.249393079234869, + "student/entropy": 0.054577818668137 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.266666666666667, + "completions/max_terminated_length": 8.266666666666667, + "completions/mean_length": 1.075000031789144, + "completions/mean_terminated_length": 1.075000031789144, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047981815754125516, + "epoch": 0.7838073899669616, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.15195369720459, + "kd/policy_loss": -3.4010083893934886, + "kd/rkl_advantage_mean": 75.6530655225118, + "kd/rkl_advantage_p95": 101.62298994064331, + "kd/rkl_advantage_std": 42.64483087758223, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1623058519728096e-07, + "loss": -13.604034423828125, + "num_tokens": 238495822.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0590352535247802, + "sampling/importance_sampling_ratio/mean": 1.000069542725881, + "sampling/importance_sampling_ratio/min": 0.9576397637526194, + "sampling/sampling_logp_difference/max": 0.10372597576739888, + "sampling/sampling_logp_difference/mean": 0.0014050467230845242, + "step": 20640, + "step_time": 4.19440633586831, + "student/entropy": 0.047981815754125516 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.633333333333334, + "completions/max_terminated_length": 6.633333333333334, + "completions/mean_length": 1.0588889161745707, + "completions/mean_terminated_length": 1.0588889161745707, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.038743013190105556, + "epoch": 0.7849466448942392, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 8.686654090881348, + "kd/policy_loss": -2.3141125361124675, + "kd/rkl_advantage_mean": 58.11612103780111, + "kd/rkl_advantage_p95": 79.16942075888316, + "kd/rkl_advantage_std": 34.47608106633027, + "kd/ssd_loss": 0.0, + "learning_rate": 2.150913302700034e-07, + "loss": -9.256448364257812, + "num_tokens": 238806498.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0425275882085165, + "sampling/importance_sampling_ratio/mean": 1.0003070890903474, + "sampling/importance_sampling_ratio/min": 0.9868401606877645, + "sampling/sampling_logp_difference/max": 0.04230713543171684, + "sampling/sampling_logp_difference/mean": 0.0006464995569937552, + "step": 20670, + "step_time": 4.185785892167769, + "student/entropy": 0.038743013190105556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.266666666666667, + "completions/max_terminated_length": 4.266666666666667, + "completions/mean_length": 1.0361111283302307, + "completions/mean_terminated_length": 1.0361111283302307, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04106300060326854, + "epoch": 0.7860858998215168, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.900419235229492, + "kd/policy_loss": 0.8549807469050089, + "kd/rkl_advantage_mean": 24.334280920028686, + "kd/rkl_advantage_p95": 39.32124338944753, + "kd/rkl_advantage_std": 21.091743831833202, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1395207534272585e-07, + "loss": 3.4199234008789063, + "num_tokens": 239121412.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0222607652346294, + "sampling/importance_sampling_ratio/mean": 0.9998581270376842, + "sampling/importance_sampling_ratio/min": 0.9568412363529205, + "sampling/sampling_logp_difference/max": 0.061485657623658575, + "sampling/sampling_logp_difference/mean": 0.0008909437601687387, + "step": 20700, + "step_time": 4.250270591930894, + "student/entropy": 0.04106300060326854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.0630555828412374, + "completions/mean_terminated_length": 1.0630555828412374, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04724329250554244, + "epoch": 0.7872251547487943, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.390148162841797, + "kd/policy_loss": -2.813515893618266, + "kd/rkl_advantage_mean": 64.09788109461466, + "kd/rkl_advantage_p95": 84.95789025624593, + "kd/rkl_advantage_std": 35.625068762898444, + "kd/ssd_loss": 0.0, + "learning_rate": 2.128128204154483e-07, + "loss": -11.25406494140625, + "num_tokens": 239424775.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0418414990107219, + "sampling/importance_sampling_ratio/mean": 1.0000989178816477, + "sampling/importance_sampling_ratio/min": 0.9735734363396963, + "sampling/sampling_logp_difference/max": 0.05545180767464141, + "sampling/sampling_logp_difference/mean": 0.0010011581587605178, + "step": 20730, + "step_time": 4.177567125132191, + "student/entropy": 0.04724329250554244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.766666666666667, + "completions/max_terminated_length": 5.766666666666667, + "completions/mean_length": 1.0516666889190673, + "completions/mean_terminated_length": 1.0516666889190673, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04627880025655031, + "epoch": 0.7883644096760718, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.84530258178711, + "kd/policy_loss": -1.4467791040738425, + "kd/rkl_advantage_mean": 44.47877842585246, + "kd/rkl_advantage_p95": 62.47056711514791, + "kd/rkl_advantage_std": 27.441030941406886, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1167356548817071e-07, + "loss": -5.787117004394531, + "num_tokens": 239730593.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0200320442517599, + "sampling/importance_sampling_ratio/mean": 0.9999886691570282, + "sampling/importance_sampling_ratio/min": 0.9753592232863109, + "sampling/sampling_logp_difference/max": 0.04778583828980724, + "sampling/sampling_logp_difference/mean": 0.0008669778132267917, + "step": 20760, + "step_time": 4.205934001535449, + "student/entropy": 0.04627880025655031 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.5, + "completions/max_terminated_length": 7.5, + "completions/mean_length": 1.0719444672266643, + "completions/mean_terminated_length": 1.0719444672266643, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05305714275067051, + "epoch": 0.7895036646033494, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.242342948913574, + "kd/policy_loss": -2.702647360165914, + "kd/rkl_advantage_mean": 60.29582956631978, + "kd/rkl_advantage_p95": 79.1019122838974, + "kd/rkl_advantage_std": 32.58214924732844, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1053431056089316e-07, + "loss": -10.810587565104166, + "num_tokens": 240051572.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0595257997512817, + "sampling/importance_sampling_ratio/mean": 1.0001912613709767, + "sampling/importance_sampling_ratio/min": 0.9654911577701568, + "sampling/sampling_logp_difference/max": 0.07017881258701285, + "sampling/sampling_logp_difference/mean": 0.0011746615540081014, + "step": 20790, + "step_time": 4.360796196728673, + "student/entropy": 0.05305714275067051 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.2, + "completions/max_terminated_length": 8.2, + "completions/mean_length": 1.0813889225323996, + "completions/mean_terminated_length": 1.0813889225323996, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049138803190241255, + "epoch": 0.7906429195306269, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.92865753173828, + "kd/policy_loss": -4.8794789950052895, + "kd/rkl_advantage_mean": 75.8330863793691, + "kd/rkl_advantage_p95": 96.4291870355606, + "kd/rkl_advantage_std": 36.65400700867176, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0939505563361563e-07, + "loss": -19.517915852864583, + "num_tokens": 240361777.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0216503620147706, + "sampling/importance_sampling_ratio/mean": 0.9996280133724212, + "sampling/importance_sampling_ratio/min": 0.9331349452336629, + "sampling/sampling_logp_difference/max": 0.10029128598980605, + "sampling/sampling_logp_difference/mean": 0.0011837064113933593, + "step": 20820, + "step_time": 4.179409818299852, + "student/entropy": 0.049138803190241255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.2, + "completions/max_terminated_length": 5.2, + "completions/mean_length": 1.0588889122009277, + "completions/mean_terminated_length": 1.0588889122009277, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048261008132249116, + "epoch": 0.7917821744579046, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.526552200317383, + "kd/policy_loss": -0.9443299333254497, + "kd/rkl_advantage_mean": 33.07984261512756, + "kd/rkl_advantage_p95": 44.92826004823049, + "kd/rkl_advantage_std": 19.300331955154736, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0825580070633805e-07, + "loss": -3.7773193359375, + "num_tokens": 240665821.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0417169729868572, + "sampling/importance_sampling_ratio/mean": 1.000179906686147, + "sampling/importance_sampling_ratio/min": 0.9773534158865611, + "sampling/sampling_logp_difference/max": 0.051595012579734124, + "sampling/sampling_logp_difference/mean": 0.0011159561113648427, + "step": 20850, + "step_time": 4.22410828233551, + "student/entropy": 0.048261008132249116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0827778100967407, + "completions/mean_terminated_length": 1.0827778100967407, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05672077865650257, + "epoch": 0.7929214293851821, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 46.8660888671875, + "kd/policy_loss": -3.6055972655614217, + "kd/rkl_advantage_mean": 57.107708883285525, + "kd/rkl_advantage_p95": 73.97709697882334, + "kd/rkl_advantage_std": 28.198101379474004, + "kd/ssd_loss": 0.0, + "learning_rate": 2.071165457790605e-07, + "loss": -14.422390747070313, + "num_tokens": 240987431.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.030669140815735, + "sampling/importance_sampling_ratio/mean": 0.9996987402439117, + "sampling/importance_sampling_ratio/min": 0.933668710788091, + "sampling/sampling_logp_difference/max": 0.08048378952468435, + "sampling/sampling_logp_difference/mean": 0.001334489208723729, + "step": 20880, + "step_time": 4.393542481533465, + "student/entropy": 0.05672077865650257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.433333333333334, + "completions/max_terminated_length": 8.433333333333334, + "completions/mean_length": 1.084166701634725, + "completions/mean_terminated_length": 1.084166701634725, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05798310556759437, + "epoch": 0.7940606843124597, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.181913375854492, + "kd/policy_loss": -3.9971886118253073, + "kd/rkl_advantage_mean": 73.99285848935445, + "kd/rkl_advantage_p95": 96.1746556520462, + "kd/rkl_advantage_std": 38.088751979668935, + "kd/ssd_loss": 0.0, + "learning_rate": 2.059772908517829e-07, + "loss": -15.988755289713541, + "num_tokens": 241295158.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0157877961794535, + "sampling/importance_sampling_ratio/mean": 0.9993836303551992, + "sampling/importance_sampling_ratio/min": 0.9242099702358246, + "sampling/sampling_logp_difference/max": 0.10377517008843522, + "sampling/sampling_logp_difference/mean": 0.0016862171425600537, + "step": 20910, + "step_time": 4.255818668833914, + "student/entropy": 0.05798310556759437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.166666666666667, + "completions/max_terminated_length": 5.166666666666667, + "completions/mean_length": 1.0447222431500753, + "completions/mean_terminated_length": 1.0447222431500753, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04858230914299687, + "epoch": 0.7951999392397372, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.054421424865723, + "kd/policy_loss": -0.3179457386334737, + "kd/rkl_advantage_mean": 37.98426232337952, + "kd/rkl_advantage_p95": 55.97827158768972, + "kd/rkl_advantage_std": 27.059318874279658, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0483803592450535e-07, + "loss": -1.271784210205078, + "num_tokens": 241607679.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0322020769119262, + "sampling/importance_sampling_ratio/mean": 1.0001810888449352, + "sampling/importance_sampling_ratio/min": 0.9834280808766683, + "sampling/sampling_logp_difference/max": 0.049323635858794054, + "sampling/sampling_logp_difference/mean": 0.000889579647143061, + "step": 20940, + "step_time": 4.296000722368384, + "student/entropy": 0.04858230914299687 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.266666666666667, + "completions/max_terminated_length": 7.266666666666667, + "completions/mean_length": 1.05805557568868, + "completions/mean_terminated_length": 1.05805557568868, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.051198571703086296, + "epoch": 0.7963391941670148, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.846794128417969, + "kd/policy_loss": -2.0355498154958087, + "kd/rkl_advantage_mean": 58.92774767875672, + "kd/rkl_advantage_p95": 79.07837433020273, + "kd/rkl_advantage_std": 34.17747834374507, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0369878099722782e-07, + "loss": -8.142198689778645, + "num_tokens": 241917376.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0269856691360473, + "sampling/importance_sampling_ratio/mean": 1.0000730435053506, + "sampling/importance_sampling_ratio/min": 0.9767480572064717, + "sampling/sampling_logp_difference/max": 0.0525712246230493, + "sampling/sampling_logp_difference/mean": 0.0009427268460664588, + "step": 20970, + "step_time": 4.292774957166208, + "student/entropy": 0.051198571703086296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.233333333333333, + "completions/max_terminated_length": 6.233333333333333, + "completions/mean_length": 1.0755555828412373, + "completions/mean_terminated_length": 1.0755555828412373, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05182328786080082, + "epoch": 0.7974784490942923, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.352559089660645, + "kd/policy_loss": -2.914878253142039, + "kd/rkl_advantage_mean": 51.35645534197489, + "kd/rkl_advantage_p95": 67.7423631032308, + "kd/rkl_advantage_std": 26.671416531006496, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0255952606995024e-07, + "loss": -11.659513346354167, + "num_tokens": 242222544.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.008570390939712524, + "sampling/importance_sampling_ratio/max": 1.0707276701927184, + "sampling/importance_sampling_ratio/mean": 1.0003243704636893, + "sampling/importance_sampling_ratio/min": 0.9653599639733632, + "sampling/sampling_logp_difference/max": 0.08916557660947243, + "sampling/sampling_logp_difference/mean": 0.0014083733500835176, + "step": 21000, + "step_time": 4.253934961800406, + "student/entropy": 0.05182328786080082 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.033333333333333, + "completions/max_terminated_length": 6.033333333333333, + "completions/mean_length": 1.0450000166893005, + "completions/mean_terminated_length": 1.0450000166893005, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05227017387126883, + "epoch": 0.7986177040215698, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.14346981048584, + "kd/policy_loss": -0.5229524850845337, + "kd/rkl_advantage_mean": 44.11470521291097, + "kd/rkl_advantage_p95": 62.08903187115987, + "kd/rkl_advantage_std": 28.952039350072543, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0142027114267268e-07, + "loss": -2.091812515258789, + "num_tokens": 242530010.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0075692772865295, + "sampling/importance_sampling_ratio/mean": 0.9997140983740489, + "sampling/importance_sampling_ratio/min": 0.9479537506898245, + "sampling/sampling_logp_difference/max": 0.06506022329752643, + "sampling/sampling_logp_difference/mean": 0.0011179134366102518, + "step": 21030, + "step_time": 4.253379207865994, + "student/entropy": 0.05227017387126883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.8, + "completions/max_terminated_length": 6.8, + "completions/mean_length": 1.080833367506663, + "completions/mean_terminated_length": 1.080833367506663, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05536884414032102, + "epoch": 0.7997569589488475, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 68.63433074951172, + "kd/policy_loss": -4.114744822184245, + "kd/rkl_advantage_mean": 60.35406959851583, + "kd/rkl_advantage_p95": 79.30294020175934, + "kd/rkl_advantage_std": 29.95374257862568, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0028101621539513e-07, + "loss": -16.458978271484376, + "num_tokens": 242842685.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0489688595136006, + "sampling/importance_sampling_ratio/mean": 0.9998401482899983, + "sampling/importance_sampling_ratio/min": 0.930743948618571, + "sampling/sampling_logp_difference/max": 0.08643419845029712, + "sampling/sampling_logp_difference/mean": 0.001516742397022123, + "step": 21060, + "step_time": 4.27531313286842, + "student/entropy": 0.05536884414032102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.833333333333333, + "completions/max_terminated_length": 6.833333333333333, + "completions/mean_length": 1.0688889145851135, + "completions/mean_terminated_length": 1.0688889145851135, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05623750559364756, + "epoch": 0.800896213876125, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 33.820674896240234, + "kd/policy_loss": -2.635713299115499, + "kd/rkl_advantage_mean": 55.901504723231, + "kd/rkl_advantage_p95": 73.77030970255534, + "kd/rkl_advantage_std": 30.372522470355033, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9914176128811755e-07, + "loss": -10.542852783203125, + "num_tokens": 243147413.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0192188064257304, + "sampling/importance_sampling_ratio/mean": 0.9997472922007243, + "sampling/importance_sampling_ratio/min": 0.9382448931535085, + "sampling/sampling_logp_difference/max": 0.06839927625842393, + "sampling/sampling_logp_difference/mean": 0.0011541895723591248, + "step": 21090, + "step_time": 4.231152911701065, + "student/entropy": 0.05623750559364756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.9, + "completions/max_terminated_length": 6.9, + "completions/mean_length": 1.0794444759686788, + "completions/mean_terminated_length": 1.0794444759686788, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.055513386894017455, + "epoch": 0.8020354688034026, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.261889457702637, + "kd/policy_loss": -3.5764529426892597, + "kd/rkl_advantage_mean": 60.713275241851804, + "kd/rkl_advantage_p95": 79.3318036476771, + "kd/rkl_advantage_std": 31.45664085249106, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9800250636084002e-07, + "loss": -14.305808512369792, + "num_tokens": 243460323.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0327961126963296, + "sampling/importance_sampling_ratio/mean": 0.999877659479777, + "sampling/importance_sampling_ratio/min": 0.9418229391177495, + "sampling/sampling_logp_difference/max": 0.0762715767758588, + "sampling/sampling_logp_difference/mean": 0.0011789810387805725, + "step": 21120, + "step_time": 4.275036592395433, + "student/entropy": 0.055513386894017455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.9, + "completions/max_terminated_length": 6.9, + "completions/mean_length": 1.061388913790385, + "completions/mean_terminated_length": 1.061388913790385, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05444789063185453, + "epoch": 0.8031747237306801, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.47679615020752, + "kd/policy_loss": -2.3507952610651652, + "kd/rkl_advantage_mean": 59.383911021550496, + "kd/rkl_advantage_p95": 78.93593507607778, + "kd/rkl_advantage_std": 33.840331912537415, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9686325143356243e-07, + "loss": -9.403179931640626, + "num_tokens": 243757888.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0393652121225994, + "sampling/importance_sampling_ratio/mean": 1.000059845050176, + "sampling/importance_sampling_ratio/min": 0.9656261662642162, + "sampling/sampling_logp_difference/max": 0.07071732954743008, + "sampling/sampling_logp_difference/mean": 0.0011695308601095652, + "step": 21150, + "step_time": 4.1931234147681, + "student/entropy": 0.05444789063185453 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.066666666666666, + "completions/max_terminated_length": 7.066666666666666, + "completions/mean_length": 1.081944473584493, + "completions/mean_terminated_length": 1.081944473584493, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.053825030320634445, + "epoch": 0.8043139786579577, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 20.218311309814453, + "kd/policy_loss": -3.5573829650878905, + "kd/rkl_advantage_mean": 57.680433940887454, + "kd/rkl_advantage_p95": 73.84650456110636, + "kd/rkl_advantage_std": 28.21001478433609, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9572399650628488e-07, + "loss": -14.229532877604166, + "num_tokens": 244073719.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0460625966389974, + "sampling/importance_sampling_ratio/mean": 1.0000424663225809, + "sampling/importance_sampling_ratio/min": 0.9427714943885803, + "sampling/sampling_logp_difference/max": 0.09081986951641738, + "sampling/sampling_logp_difference/mean": 0.0014163440316527461, + "step": 21180, + "step_time": 4.258232422966103, + "student/entropy": 0.053825030320634445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.3, + "completions/max_terminated_length": 7.3, + "completions/mean_length": 1.0680555860201517, + "completions/mean_terminated_length": 1.0680555860201517, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05166515897338589, + "epoch": 0.8054532335852352, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.041813850402832, + "kd/policy_loss": -3.3474434038003285, + "kd/rkl_advantage_mean": 66.98674685160319, + "kd/rkl_advantage_p95": 90.20314807097117, + "kd/rkl_advantage_std": 37.437972235679624, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9458474157900732e-07, + "loss": -13.389776611328125, + "num_tokens": 244380740.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.017699100822210313, + "sampling/importance_sampling_ratio/max": 1.0261696894963583, + "sampling/importance_sampling_ratio/mean": 0.9996022899945577, + "sampling/importance_sampling_ratio/min": 0.9207017242908477, + "sampling/sampling_logp_difference/max": 0.10950165999432405, + "sampling/sampling_logp_difference/mean": 0.0015186813417434073, + "step": 21210, + "step_time": 4.284350487832368, + "student/entropy": 0.05166515897338589 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 22.466666666666665, + "completions/max_terminated_length": 5.433333333333334, + "completions/mean_length": 1.187500031789144, + "completions/mean_terminated_length": 1.04555557568868, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04800773411989212, + "epoch": 0.8065924885125129, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.27269172668457, + "kd/policy_loss": -0.7029859562714894, + "kd/rkl_advantage_mean": 45.572673145929976, + "kd/rkl_advantage_p95": 62.39380674362182, + "kd/rkl_advantage_std": 27.555360105633735, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9344548665172977e-07, + "loss": -2.811944071451823, + "num_tokens": 244701655.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.1119187076886494, + "sampling/importance_sampling_ratio/mean": 1.0007576922575632, + "sampling/importance_sampling_ratio/min": 0.975382783015569, + "sampling/sampling_logp_difference/max": 0.08780941710186502, + "sampling/sampling_logp_difference/mean": 0.001069504972838331, + "step": 21240, + "step_time": 4.309437400235523, + "student/entropy": 0.04800773411989212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.566666666666666, + "completions/max_terminated_length": 7.566666666666666, + "completions/mean_length": 1.0872222542762757, + "completions/mean_terminated_length": 1.0872222542762757, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04915220315257708, + "epoch": 0.8077317434397904, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.703693389892578, + "kd/policy_loss": -3.376502434412638, + "kd/rkl_advantage_mean": 65.22546582221985, + "kd/rkl_advantage_p95": 84.93628555933634, + "kd/rkl_advantage_std": 34.36697493394216, + "kd/ssd_loss": 0.0, + "learning_rate": 1.923062317244522e-07, + "loss": -13.506013997395833, + "num_tokens": 245010681.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.02068643172582, + "sampling/importance_sampling_ratio/mean": 0.9995338340600332, + "sampling/importance_sampling_ratio/min": 0.9285329202810924, + "sampling/sampling_logp_difference/max": 0.08612736936969062, + "sampling/sampling_logp_difference/mean": 0.0011406989831205768, + "step": 21270, + "step_time": 4.151472043500689, + "student/entropy": 0.04915220315257708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.933333333333334, + "completions/max_terminated_length": 5.933333333333334, + "completions/mean_length": 1.0480555733044943, + "completions/mean_terminated_length": 1.0480555733044943, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04830132313072681, + "epoch": 0.8088709983670679, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.105696678161621, + "kd/policy_loss": -1.1315462867418924, + "kd/rkl_advantage_mean": 44.92161021232605, + "kd/rkl_advantage_p95": 62.17518235047658, + "kd/rkl_advantage_std": 27.961248231927552, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9116697679717463e-07, + "loss": -4.52618408203125, + "num_tokens": 245314422.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0217782894770304, + "sampling/importance_sampling_ratio/mean": 0.9999378025531769, + "sampling/importance_sampling_ratio/min": 0.9662684122721354, + "sampling/sampling_logp_difference/max": 0.05143664314722021, + "sampling/sampling_logp_difference/mean": 0.0009160118536480392, + "step": 21300, + "step_time": 4.201335615200515, + "student/entropy": 0.04830132313072681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.733333333333333, + "completions/max_terminated_length": 6.733333333333333, + "completions/mean_length": 1.0611111362775167, + "completions/mean_terminated_length": 1.0611111362775167, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05539894069855412, + "epoch": 0.8100102532943455, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.287677764892578, + "kd/policy_loss": -2.2658300161361695, + "kd/rkl_advantage_mean": 54.66187701225281, + "kd/rkl_advantage_p95": 72.97330315113068, + "kd/rkl_advantage_std": 30.917094778021177, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9002772186989707e-07, + "loss": -9.063319905598958, + "num_tokens": 245617682.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0390010952949524, + "sampling/importance_sampling_ratio/mean": 0.9999450782934824, + "sampling/importance_sampling_ratio/min": 0.9540558417638143, + "sampling/sampling_logp_difference/max": 0.09251406689484915, + "sampling/sampling_logp_difference/mean": 0.0015539797944560025, + "step": 21330, + "step_time": 4.173989859265081, + "student/entropy": 0.05539894069855412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.733333333333333, + "completions/max_terminated_length": 6.733333333333333, + "completions/mean_length": 1.071666701634725, + "completions/mean_terminated_length": 1.071666701634725, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049812024210890135, + "epoch": 0.811149508221623, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.0577392578125, + "kd/policy_loss": -3.233787206808726, + "kd/rkl_advantage_mean": 58.80478432973226, + "kd/rkl_advantage_p95": 79.25531595547994, + "kd/rkl_advantage_std": 31.971226994196574, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8888846694261952e-07, + "loss": -12.935150146484375, + "num_tokens": 245936948.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0412131547927856, + "sampling/importance_sampling_ratio/mean": 0.9998112261295319, + "sampling/importance_sampling_ratio/min": 0.94731857975324, + "sampling/sampling_logp_difference/max": 0.08497679162149628, + "sampling/sampling_logp_difference/mean": 0.0013030146258339907, + "step": 21360, + "step_time": 4.307576187265416, + "student/entropy": 0.049812024210890135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.733333333333333, + "completions/max_terminated_length": 7.733333333333333, + "completions/mean_length": 1.0780555844306945, + "completions/mean_terminated_length": 1.0780555844306945, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048341246259709196, + "epoch": 0.8122887631489006, + "eval/gt_acc_batch": 0.002222222338120143, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.047372817993164, + "kd/policy_loss": -3.660180759429932, + "kd/rkl_advantage_mean": 69.07141963640849, + "kd/rkl_advantage_p95": 90.22144196828206, + "kd/rkl_advantage_std": 36.494564115504424, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8774921201534196e-07, + "loss": -14.640724690755208, + "num_tokens": 246241133.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.002222222338120143, + "rewards/gt_logging_reward/std": 0.024343226353327432, + "sampling/importance_sampling_ratio/max": 1.0263592958450318, + "sampling/importance_sampling_ratio/mean": 0.9995964785416921, + "sampling/importance_sampling_ratio/min": 0.9414327959219615, + "sampling/sampling_logp_difference/max": 0.07912864943500608, + "sampling/sampling_logp_difference/mean": 0.0012377972384759535, + "step": 21390, + "step_time": 4.2796305220976745, + "student/entropy": 0.048341246259709196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.533333333333333, + "completions/max_terminated_length": 6.533333333333333, + "completions/mean_length": 1.061666695276896, + "completions/mean_terminated_length": 1.061666695276896, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04673161127914985, + "epoch": 0.8134280180761781, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.689571380615234, + "kd/policy_loss": -2.533141509691874, + "kd/rkl_advantage_mean": 57.25093264579773, + "kd/rkl_advantage_p95": 79.05126924514771, + "kd/rkl_advantage_std": 33.70642575720946, + "kd/ssd_loss": 0.0, + "learning_rate": 1.866099570880644e-07, + "loss": -10.132567342122396, + "num_tokens": 246544243.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.017699100822210313, + "sampling/importance_sampling_ratio/max": 1.0231959700584412, + "sampling/importance_sampling_ratio/mean": 0.9999092201391856, + "sampling/importance_sampling_ratio/min": 0.970349399248759, + "sampling/sampling_logp_difference/max": 0.0538476328831166, + "sampling/sampling_logp_difference/mean": 0.0009314078347718653, + "step": 21420, + "step_time": 4.254865256731864, + "student/entropy": 0.04673161127914985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.166666666666667, + "completions/max_terminated_length": 7.166666666666667, + "completions/mean_length": 1.0722222487131754, + "completions/mean_terminated_length": 1.0722222487131754, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047035163703064126, + "epoch": 0.8145672730034558, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 252.51025390625, + "kd/policy_loss": -2.8879656195640564, + "kd/rkl_advantage_mean": 59.60257703463237, + "kd/rkl_advantage_p95": 79.18943458398184, + "kd/rkl_advantage_std": 32.3559478243192, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8547070216078682e-07, + "loss": -11.551863606770834, + "num_tokens": 246851479.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0432798107465109, + "sampling/importance_sampling_ratio/mean": 0.9998685081799825, + "sampling/importance_sampling_ratio/min": 0.9457688450813293, + "sampling/sampling_logp_difference/max": 0.0764964547008276, + "sampling/sampling_logp_difference/mean": 0.0011484603591573736, + "step": 21450, + "step_time": 4.195893807065052, + "student/entropy": 0.047035163703064126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.466666666666667, + "completions/max_terminated_length": 4.466666666666667, + "completions/mean_length": 1.0450000206629435, + "completions/mean_terminated_length": 1.0450000206629435, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0456460598235329, + "epoch": 0.8157065279307333, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.355521202087402, + "kd/policy_loss": 0.40834958155949913, + "kd/rkl_advantage_mean": 28.529499705632528, + "kd/rkl_advantage_p95": 44.8464950243632, + "kd/rkl_advantage_std": 23.224522507190706, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8433144723350927e-07, + "loss": 1.633399200439453, + "num_tokens": 247156089.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0220272302627564, + "sampling/importance_sampling_ratio/mean": 0.9997516791025798, + "sampling/importance_sampling_ratio/min": 0.94752057492733, + "sampling/sampling_logp_difference/max": 0.08674272165323298, + "sampling/sampling_logp_difference/mean": 0.0011414773548798014, + "step": 21480, + "step_time": 4.240601038832392, + "student/entropy": 0.0456460598235329 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.7, + "completions/max_terminated_length": 7.7, + "completions/mean_length": 1.084166697661082, + "completions/mean_terminated_length": 1.084166697661082, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05547654718781511, + "epoch": 0.8168457828580109, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.717747688293457, + "kd/policy_loss": -3.3335710883140566, + "kd/rkl_advantage_mean": 64.72371457417806, + "kd/rkl_advantage_p95": 84.2414297580719, + "kd/rkl_advantage_std": 33.796361870567004, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8319219230623174e-07, + "loss": -13.334285481770833, + "num_tokens": 247462656.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0415147384007772, + "sampling/importance_sampling_ratio/mean": 0.9996658583482106, + "sampling/importance_sampling_ratio/min": 0.9201799074808756, + "sampling/sampling_logp_difference/max": 0.10770935125959416, + "sampling/sampling_logp_difference/mean": 0.0015046101519449924, + "step": 21510, + "step_time": 4.200618800835218, + "student/entropy": 0.05547654718781511 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.133333333333334, + "completions/max_terminated_length": 7.133333333333334, + "completions/mean_length": 1.0708333532015482, + "completions/mean_terminated_length": 1.0708333532015482, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049606022300819556, + "epoch": 0.8179850377852884, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.358380317687988, + "kd/policy_loss": -1.9124063849449158, + "kd/rkl_advantage_mean": 55.383901580174765, + "kd/rkl_advantage_p95": 73.53954956531524, + "kd/rkl_advantage_std": 31.28908865004778, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8205293737895416e-07, + "loss": -7.649626668294271, + "num_tokens": 247770327.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0126250465710958, + "sampling/importance_sampling_ratio/mean": 0.999569547176361, + "sampling/importance_sampling_ratio/min": 0.937383179863294, + "sampling/sampling_logp_difference/max": 0.07224327653336028, + "sampling/sampling_logp_difference/mean": 0.0010952503983086596, + "step": 21540, + "step_time": 4.262653397402028, + "student/entropy": 0.049606022300819556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.366666666666666, + "completions/max_terminated_length": 7.366666666666666, + "completions/mean_length": 1.0777778108914693, + "completions/mean_terminated_length": 1.0777778108914693, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.050200322673966485, + "epoch": 0.8191242927125659, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 42.7221565246582, + "kd/policy_loss": -3.0621110081672667, + "kd/rkl_advantage_mean": 64.14593906402588, + "kd/rkl_advantage_p95": 85.13798121611278, + "kd/rkl_advantage_std": 35.275785712401074, + "kd/ssd_loss": 0.0, + "learning_rate": 1.809136824516766e-07, + "loss": -12.248447672526042, + "num_tokens": 248082799.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0234899799029031, + "sampling/importance_sampling_ratio/mean": 0.9998729129632314, + "sampling/importance_sampling_ratio/min": 0.955097762743632, + "sampling/sampling_logp_difference/max": 0.06993204954390725, + "sampling/sampling_logp_difference/mean": 0.0011498158137934903, + "step": 21570, + "step_time": 4.301265954671544, + "student/entropy": 0.050200322673966485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.076111133893331, + "completions/mean_terminated_length": 1.076111133893331, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05165403643622994, + "epoch": 0.8202635476398435, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.588831901550293, + "kd/policy_loss": -2.5522868633270264, + "kd/rkl_advantage_mean": 52.77848869959514, + "kd/rkl_advantage_p95": 68.38288417657216, + "kd/rkl_advantage_std": 27.25390192915996, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7977442752439904e-07, + "loss": -10.20915018717448, + "num_tokens": 248397857.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0212488849957784, + "sampling/importance_sampling_ratio/mean": 0.9996507048606873, + "sampling/importance_sampling_ratio/min": 0.9308300892512004, + "sampling/sampling_logp_difference/max": 0.08136866121397664, + "sampling/sampling_logp_difference/mean": 0.0014400769403437152, + "step": 21600, + "step_time": 4.267010381265815, + "student/entropy": 0.05165403643622994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.466666666666667, + "completions/max_terminated_length": 6.466666666666667, + "completions/mean_length": 1.0536111354827882, + "completions/mean_terminated_length": 1.0536111354827882, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.051892343908548355, + "epoch": 0.8214028025671211, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 32.79694747924805, + "kd/policy_loss": -1.957354478041331, + "kd/rkl_advantage_mean": 54.318836800257365, + "kd/rkl_advantage_p95": 73.69086266358694, + "kd/rkl_advantage_std": 32.21685976237059, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7863517259712146e-07, + "loss": -7.829420471191407, + "num_tokens": 248706106.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0113232254981994, + "sampling/importance_sampling_ratio/mean": 0.9999456862608592, + "sampling/importance_sampling_ratio/min": 0.9704085886478424, + "sampling/sampling_logp_difference/max": 0.05295295041675369, + "sampling/sampling_logp_difference/mean": 0.000998906628228724, + "step": 21630, + "step_time": 4.204541360600463, + "student/entropy": 0.051892343908548355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.733333333333333, + "completions/max_terminated_length": 7.733333333333333, + "completions/mean_length": 1.068333355585734, + "completions/mean_terminated_length": 1.068333355585734, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.051426569496591884, + "epoch": 0.8225420574943987, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.317487716674805, + "kd/policy_loss": -2.8386759757995605, + "kd/rkl_advantage_mean": 64.93762008349101, + "kd/rkl_advantage_p95": 84.96999082565307, + "kd/rkl_advantage_std": 35.18327874143918, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7749591766984393e-07, + "loss": -11.354705810546875, + "num_tokens": 249008416.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0414413412412007, + "sampling/importance_sampling_ratio/mean": 0.9999238888422648, + "sampling/importance_sampling_ratio/min": 0.9377439399560292, + "sampling/sampling_logp_difference/max": 0.08751702613662929, + "sampling/sampling_logp_difference/mean": 0.0012956708640558645, + "step": 21660, + "step_time": 4.271686374669661, + "student/entropy": 0.051426569496591884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.666666666666667, + "completions/max_terminated_length": 4.666666666666667, + "completions/mean_length": 1.050000019868215, + "completions/mean_terminated_length": 1.050000019868215, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0530737545962135, + "epoch": 0.8236813124216762, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.160225868225098, + "kd/policy_loss": -0.21979012886683147, + "kd/rkl_advantage_mean": 27.809279998143513, + "kd/rkl_advantage_p95": 39.15796269575755, + "kd/rkl_advantage_std": 18.163424090544382, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7635666274256635e-07, + "loss": -0.879161262512207, + "num_tokens": 249311820.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.059391689300537, + "sampling/importance_sampling_ratio/mean": 1.0002743701140087, + "sampling/importance_sampling_ratio/min": 0.9637307286262512, + "sampling/sampling_logp_difference/max": 0.05144489570520818, + "sampling/sampling_logp_difference/mean": 0.0011056241656964025, + "step": 21690, + "step_time": 4.2269539873334, + "student/entropy": 0.0530737545962135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.033333333333333, + "completions/max_terminated_length": 8.033333333333333, + "completions/mean_length": 1.078611139456431, + "completions/mean_terminated_length": 1.078611139456431, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05622192807495594, + "epoch": 0.8248205673489538, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 41.34593963623047, + "kd/policy_loss": -3.805747628211975, + "kd/rkl_advantage_mean": 73.82394822438557, + "kd/rkl_advantage_p95": 96.10989676316579, + "kd/rkl_advantage_std": 39.27158792118232, + "kd/ssd_loss": 0.0, + "learning_rate": 1.752174078152888e-07, + "loss": -15.222989908854167, + "num_tokens": 249617847.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.03722802400589, + "sampling/importance_sampling_ratio/mean": 0.9996593058109283, + "sampling/importance_sampling_ratio/min": 0.9188563525676727, + "sampling/sampling_logp_difference/max": 0.11719939263227085, + "sampling/sampling_logp_difference/mean": 0.0015774020042348031, + "step": 21720, + "step_time": 4.232075209333076, + "student/entropy": 0.05622192807495594 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.566666666666666, + "completions/max_terminated_length": 7.566666666666666, + "completions/mean_length": 1.0816667040189107, + "completions/mean_terminated_length": 1.0816667040189107, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049451622956742845, + "epoch": 0.8259598222762313, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 33.49643325805664, + "kd/policy_loss": -3.6591817855834963, + "kd/rkl_advantage_mean": 68.38356714248657, + "kd/rkl_advantage_p95": 90.52842174371084, + "kd/rkl_advantage_std": 36.834198792775474, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7407815288801124e-07, + "loss": -14.636724853515625, + "num_tokens": 249942877.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0177954991658529, + "sampling/importance_sampling_ratio/mean": 0.9993123630682628, + "sampling/importance_sampling_ratio/min": 0.9072138011455536, + "sampling/sampling_logp_difference/max": 0.11627013188165923, + "sampling/sampling_logp_difference/mean": 0.001467884141796579, + "step": 21750, + "step_time": 4.387786673200996, + "student/entropy": 0.049451622956742845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 23.566666666666666, + "completions/max_terminated_length": 6.9, + "completions/mean_length": 1.215000037352244, + "completions/mean_terminated_length": 1.0731232722600301, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04771189466118812, + "epoch": 0.827099077203509, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 38.101863861083984, + "kd/policy_loss": -2.481389359633128, + "kd/rkl_advantage_mean": 56.60228702227275, + "kd/rkl_advantage_p95": 73.31413151423136, + "kd/rkl_advantage_std": 28.77721662223339, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7293889796073366e-07, + "loss": -9.925558471679688, + "num_tokens": 250244091.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0214911937713622, + "sampling/importance_sampling_ratio/mean": 0.9997917334238688, + "sampling/importance_sampling_ratio/min": 0.9405899246533712, + "sampling/sampling_logp_difference/max": 0.06519922285030286, + "sampling/sampling_logp_difference/mean": 0.001019326011495044, + "step": 21780, + "step_time": 4.307785242368603, + "student/entropy": 0.04771189466118812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.833333333333334, + "completions/max_terminated_length": 8.833333333333334, + "completions/mean_length": 1.0766666928927104, + "completions/mean_terminated_length": 1.0766666928927104, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.045543566532433036, + "epoch": 0.8282383321307865, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 35.57807159423828, + "kd/policy_loss": -3.6141640742619834, + "kd/rkl_advantage_mean": 74.70629256566366, + "kd/rkl_advantage_p95": 96.15853612422943, + "kd/rkl_advantage_std": 38.81681352655093, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7179964303345613e-07, + "loss": -14.456656901041667, + "num_tokens": 250546223.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0538066546122233, + "sampling/importance_sampling_ratio/mean": 1.0001441260178885, + "sampling/importance_sampling_ratio/min": 0.9510208209355672, + "sampling/sampling_logp_difference/max": 0.08516451416071505, + "sampling/sampling_logp_difference/mean": 0.001160991886475434, + "step": 21810, + "step_time": 4.303463061365377, + "student/entropy": 0.045543566532433036 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.933333333333334, + "completions/max_terminated_length": 4.933333333333334, + "completions/mean_length": 1.0425000190734863, + "completions/mean_terminated_length": 1.0425000190734863, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.043239068674544495, + "epoch": 0.8293775870580641, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.942525863647461, + "kd/policy_loss": 0.29205259879430134, + "kd/rkl_advantage_mean": 30.845564874013267, + "kd/rkl_advantage_p95": 45.01947871049245, + "kd/rkl_advantage_std": 22.366701542337736, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7066038810617854e-07, + "loss": 1.1682108561197917, + "num_tokens": 250848088.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0255093415578207, + "sampling/importance_sampling_ratio/mean": 1.0001475016276042, + "sampling/importance_sampling_ratio/min": 0.976369837919871, + "sampling/sampling_logp_difference/max": 0.03917868194791178, + "sampling/sampling_logp_difference/mean": 0.0008115937176626176, + "step": 21840, + "step_time": 4.193130056101169, + "student/entropy": 0.043239068674544495 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.933333333333334, + "completions/max_terminated_length": 5.933333333333334, + "completions/mean_length": 1.061388913790385, + "completions/mean_terminated_length": 1.061388913790385, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.051172134652733806, + "epoch": 0.8305168419853416, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.80060386657715, + "kd/policy_loss": -1.8182763457298279, + "kd/rkl_advantage_mean": 45.761820220947264, + "kd/rkl_advantage_p95": 62.14406164487203, + "kd/rkl_advantage_std": 26.37188433110714, + "kd/ssd_loss": 0.0, + "learning_rate": 1.69521133178901e-07, + "loss": -7.27310536702474, + "num_tokens": 251145525.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0386563022931417, + "sampling/importance_sampling_ratio/mean": 1.0001321951548259, + "sampling/importance_sampling_ratio/min": 0.9574150204658508, + "sampling/sampling_logp_difference/max": 0.06754852168572446, + "sampling/sampling_logp_difference/mean": 0.0010132671372654538, + "step": 21870, + "step_time": 4.179825922064871, + "student/entropy": 0.051172134652733806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.4, + "completions/max_terminated_length": 6.4, + "completions/mean_length": 1.0552778045336406, + "completions/mean_terminated_length": 1.0552778045336406, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05760890462746223, + "epoch": 0.8316560969126191, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.365922927856445, + "kd/policy_loss": -1.9375239253044128, + "kd/rkl_advantage_mean": 54.10922916730245, + "kd/rkl_advantage_p95": 73.1930739402771, + "kd/rkl_advantage_std": 32.04157292544842, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6838187825162343e-07, + "loss": -7.750092569986979, + "num_tokens": 251446132.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0070154825846354, + "sampling/importance_sampling_ratio/mean": 0.9998928944269816, + "sampling/importance_sampling_ratio/min": 0.9613981584707896, + "sampling/sampling_logp_difference/max": 0.05406318937893957, + "sampling/sampling_logp_difference/mean": 0.0009001175737163672, + "step": 21900, + "step_time": 4.208371023370516, + "student/entropy": 0.05760890462746223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.6, + "completions/max_terminated_length": 6.6, + "completions/mean_length": 1.0538889129956563, + "completions/mean_terminated_length": 1.0538889129956563, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05500600763286154, + "epoch": 0.8327953518398967, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.200637817382812, + "kd/policy_loss": -2.019814097881317, + "kd/rkl_advantage_mean": 54.8870479742686, + "kd/rkl_advantage_p95": 74.02023255030313, + "kd/rkl_advantage_std": 31.89814397394657, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6724262332434588e-07, + "loss": -8.079257710774739, + "num_tokens": 251763766.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0217325806617736, + "sampling/importance_sampling_ratio/mean": 0.9999459683895111, + "sampling/importance_sampling_ratio/min": 0.9584628562132518, + "sampling/sampling_logp_difference/max": 0.06761215125831464, + "sampling/sampling_logp_difference/mean": 0.0010861672771473726, + "step": 21930, + "step_time": 4.290139343899985, + "student/entropy": 0.05500600763286154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.0691666920979819, + "completions/mean_terminated_length": 1.0691666920979819, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.053162959838906926, + "epoch": 0.8339346067671742, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 158.45729064941406, + "kd/policy_loss": -2.613387993971507, + "kd/rkl_advantage_mean": 62.93832392692566, + "kd/rkl_advantage_p95": 84.64133922259013, + "kd/rkl_advantage_std": 36.017176692684494, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6610336839706832e-07, + "loss": -10.453551228841146, + "num_tokens": 252063983.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.034377884864807, + "sampling/importance_sampling_ratio/mean": 1.0001678109169005, + "sampling/importance_sampling_ratio/min": 0.957738733291626, + "sampling/sampling_logp_difference/max": 0.06822653183092674, + "sampling/sampling_logp_difference/mean": 0.0010059644914387415, + "step": 21960, + "step_time": 4.192256025034294, + "student/entropy": 0.053162959838906926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.0561111370722454, + "completions/mean_terminated_length": 1.0561111370722454, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05087375690539678, + "epoch": 0.8350738616944519, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.772076606750488, + "kd/policy_loss": -1.5289586385091145, + "kd/rkl_advantage_mean": 48.78533047040303, + "kd/rkl_advantage_p95": 67.0808756351471, + "kd/rkl_advantage_std": 29.912439634402592, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6496411346979074e-07, + "loss": -6.115836588541667, + "num_tokens": 252378985.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0090039054552713, + "sampling/importance_sampling_ratio/mean": 0.9996163805325826, + "sampling/importance_sampling_ratio/min": 0.9446050961812337, + "sampling/sampling_logp_difference/max": 0.06177402374645074, + "sampling/sampling_logp_difference/mean": 0.0010858997363053884, + "step": 21990, + "step_time": 4.387326257035117, + "student/entropy": 0.05087375690539678 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.666666666666666, + "completions/max_terminated_length": 8.666666666666666, + "completions/mean_length": 1.08555558125178, + "completions/mean_terminated_length": 1.08555558125178, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05334116763745745, + "epoch": 0.8362131166217294, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.706389427185059, + "kd/policy_loss": -3.842676643530528, + "kd/rkl_advantage_mean": 73.95425769488017, + "kd/rkl_advantage_p95": 96.43923650582632, + "kd/rkl_advantage_std": 38.51108776827653, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6382485854251318e-07, + "loss": -15.370706176757812, + "num_tokens": 252681245.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0417239507039389, + "sampling/importance_sampling_ratio/mean": 0.999872891108195, + "sampling/importance_sampling_ratio/min": 0.9321342587471009, + "sampling/sampling_logp_difference/max": 0.08524769015299777, + "sampling/sampling_logp_difference/mean": 0.0014212319307262077, + "step": 22020, + "step_time": 4.257403238699286, + "student/entropy": 0.05334116763745745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.566666666666666, + "completions/max_terminated_length": 7.566666666666666, + "completions/mean_length": 1.0736111402511597, + "completions/mean_terminated_length": 1.0736111402511597, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04601092484469215, + "epoch": 0.837352371549007, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.15238094329834, + "kd/policy_loss": -3.676264560222626, + "kd/rkl_advantage_mean": 65.40896639823913, + "kd/rkl_advantage_p95": 84.790003546079, + "kd/rkl_advantage_std": 33.24798363049825, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6268560361523563e-07, + "loss": -14.705057779947916, + "num_tokens": 252982174.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.041836142539978, + "sampling/importance_sampling_ratio/mean": 0.9998555680116018, + "sampling/importance_sampling_ratio/min": 0.9334681570529938, + "sampling/sampling_logp_difference/max": 0.08633326138369739, + "sampling/sampling_logp_difference/mean": 0.0011317873616159583, + "step": 22050, + "step_time": 4.191046683033346, + "student/entropy": 0.04601092484469215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.066666666666666, + "completions/max_terminated_length": 4.066666666666666, + "completions/mean_length": 1.0336111267407735, + "completions/mean_terminated_length": 1.0336111267407735, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04505622504899899, + "epoch": 0.8384916264762845, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.740032196044922, + "kd/policy_loss": 1.433042307694753, + "kd/rkl_advantage_mean": 20.538707049687705, + "kd/rkl_advantage_p95": 32.967626857757566, + "kd/rkl_advantage_std": 19.082575010259948, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6154634868795807e-07, + "loss": 5.732169596354167, + "num_tokens": 253288175.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.022705332438151, + "sampling/importance_sampling_ratio/mean": 1.0000716030597687, + "sampling/importance_sampling_ratio/min": 0.9750091910362244, + "sampling/sampling_logp_difference/max": 0.03311954511639972, + "sampling/sampling_logp_difference/mean": 0.0007575486461670759, + "step": 22080, + "step_time": 4.231810428096408, + "student/entropy": 0.04505622504899899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.166666666666667, + "completions/max_terminated_length": 7.166666666666667, + "completions/mean_length": 1.0708333611488343, + "completions/mean_terminated_length": 1.0708333611488343, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05418849286312858, + "epoch": 0.8396308814035621, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 61.41021728515625, + "kd/policy_loss": -2.272800290584564, + "kd/rkl_advantage_mean": 55.33434036572774, + "kd/rkl_advantage_p95": 73.33374092578887, + "kd/rkl_advantage_std": 30.93195436447859, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6040709376068051e-07, + "loss": -9.091200764973959, + "num_tokens": 253593334.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0404202580451964, + "sampling/importance_sampling_ratio/mean": 1.000182859102885, + "sampling/importance_sampling_ratio/min": 0.9714600741863251, + "sampling/sampling_logp_difference/max": 0.05688336553672949, + "sampling/sampling_logp_difference/mean": 0.0010680568433599547, + "step": 22110, + "step_time": 4.302682430698769, + "student/entropy": 0.05418849286312858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.066666666666666, + "completions/max_terminated_length": 7.066666666666666, + "completions/mean_length": 1.0872222582499187, + "completions/mean_terminated_length": 1.0872222582499187, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.057701312781622015, + "epoch": 0.8407701363308396, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 41.28787612915039, + "kd/policy_loss": -3.3226613203684487, + "kd/rkl_advantage_mean": 60.95558177630107, + "kd/rkl_advantage_p95": 79.04678533871969, + "kd/rkl_advantage_std": 31.53297125498454, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5926783883340296e-07, + "loss": -13.290644327799479, + "num_tokens": 253898024.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.043286629517873, + "sampling/importance_sampling_ratio/mean": 1.0000787734985352, + "sampling/importance_sampling_ratio/min": 0.9567647258440654, + "sampling/sampling_logp_difference/max": 0.07278307486946384, + "sampling/sampling_logp_difference/mean": 0.0013053306475436935, + "step": 22140, + "step_time": 4.257447943898054, + "student/entropy": 0.057701312781622015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.0, + "completions/max_terminated_length": 8.0, + "completions/mean_length": 1.0680555860201517, + "completions/mean_terminated_length": 1.0680555860201517, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05249827976028124, + "epoch": 0.8419093912581171, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.071762084960938, + "kd/policy_loss": -3.4889647722244264, + "kd/rkl_advantage_mean": 72.67831977208455, + "kd/rkl_advantage_p95": 95.65493871370951, + "kd/rkl_advantage_std": 39.82235713203748, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5812858390612538e-07, + "loss": -13.955857340494791, + "num_tokens": 254199997.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0544801195462545, + "sampling/importance_sampling_ratio/mean": 1.000170385837555, + "sampling/importance_sampling_ratio/min": 0.9555929223696391, + "sampling/sampling_logp_difference/max": 0.09175545490191629, + "sampling/sampling_logp_difference/mean": 0.001302648207153349, + "step": 22170, + "step_time": 4.30928630400061, + "student/entropy": 0.05249827976028124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.933333333333334, + "completions/max_terminated_length": 5.933333333333334, + "completions/mean_length": 1.0688889185587565, + "completions/mean_terminated_length": 1.0688889185587565, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047786405744651955, + "epoch": 0.8430486461853948, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 41.79848861694336, + "kd/policy_loss": -2.7465879480044046, + "kd/rkl_advantage_mean": 47.58262521425883, + "kd/rkl_advantage_p95": 62.496175305048624, + "kd/rkl_advantage_std": 24.436141620079677, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5698932897884782e-07, + "loss": -10.986349487304688, + "num_tokens": 254511669.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0099047780036927, + "sampling/importance_sampling_ratio/mean": 0.9997567137082418, + "sampling/importance_sampling_ratio/min": 0.9554344018300375, + "sampling/sampling_logp_difference/max": 0.04688365482725203, + "sampling/sampling_logp_difference/mean": 0.0008426427622907795, + "step": 22200, + "step_time": 4.251871991201187, + "student/entropy": 0.047786405744651955 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.766666666666667, + "completions/max_terminated_length": 6.766666666666667, + "completions/mean_length": 1.0611111402511597, + "completions/mean_terminated_length": 1.0611111402511597, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04964466483021776, + "epoch": 0.8441879011126723, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.229161262512207, + "kd/policy_loss": -2.6909947713216145, + "kd/rkl_advantage_mean": 59.197432629267375, + "kd/rkl_advantage_p95": 78.60527994632722, + "kd/rkl_advantage_std": 33.28654415955146, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5585007405157026e-07, + "loss": -10.763980102539062, + "num_tokens": 254805921.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.045474370320638, + "sampling/importance_sampling_ratio/mean": 1.0000547309716543, + "sampling/importance_sampling_ratio/min": 0.9563183188438416, + "sampling/sampling_logp_difference/max": 0.07244639927521349, + "sampling/sampling_logp_difference/mean": 0.0010931727922676752, + "step": 22230, + "step_time": 4.199586274535007, + "student/entropy": 0.04964466483021776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.5, + "completions/max_terminated_length": 8.5, + "completions/mean_length": 1.0813889185587564, + "completions/mean_terminated_length": 1.0813889185587564, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048991316153357424, + "epoch": 0.8453271560399499, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.2352294921875, + "kd/policy_loss": -4.1025179346402485, + "kd/rkl_advantage_mean": 74.4670123577118, + "kd/rkl_advantage_p95": 96.3851559082667, + "kd/rkl_advantage_std": 38.02779893825451, + "kd/ssd_loss": 0.0, + "learning_rate": 1.547108191242927e-07, + "loss": -16.41007283528646, + "num_tokens": 255113982.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0436809976895651, + "sampling/importance_sampling_ratio/mean": 0.9996872742970785, + "sampling/importance_sampling_ratio/min": 0.9174299359321594, + "sampling/sampling_logp_difference/max": 0.10930204195901752, + "sampling/sampling_logp_difference/mean": 0.0014260056326747873, + "step": 22260, + "step_time": 4.300218412799101, + "student/entropy": 0.048991316153357424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.0, + "completions/max_terminated_length": 5.0, + "completions/mean_length": 1.0519444704055787, + "completions/mean_terminated_length": 1.0519444704055787, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.044119716590891284, + "epoch": 0.8464664109672274, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.359978675842285, + "kd/policy_loss": -0.9962990641593933, + "kd/rkl_advantage_mean": 36.003663857777916, + "kd/rkl_advantage_p95": 50.83690954844157, + "kd/rkl_advantage_std": 23.05052527288596, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5357156419701515e-07, + "loss": -3.9851949055989584, + "num_tokens": 255430073.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.032350242137909, + "sampling/importance_sampling_ratio/mean": 0.9998652835687002, + "sampling/importance_sampling_ratio/min": 0.9479294200738271, + "sampling/sampling_logp_difference/max": 0.06615916675267121, + "sampling/sampling_logp_difference/mean": 0.0011989043162126715, + "step": 22290, + "step_time": 4.36236042129943, + "student/entropy": 0.044119716590891284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.633333333333334, + "completions/max_terminated_length": 6.633333333333334, + "completions/mean_length": 1.0658333579699197, + "completions/mean_terminated_length": 1.0658333579699197, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047493799372265735, + "epoch": 0.847605665894505, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.71818733215332, + "kd/policy_loss": -2.845424211025238, + "kd/rkl_advantage_mean": 55.58369650840759, + "kd/rkl_advantage_p95": 73.54716565608979, + "kd/rkl_advantage_std": 29.937193674842515, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5243230926973757e-07, + "loss": -11.381697591145834, + "num_tokens": 255732838.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.038522756099701, + "sampling/importance_sampling_ratio/mean": 0.9999177654584249, + "sampling/importance_sampling_ratio/min": 0.9531352778275808, + "sampling/sampling_logp_difference/max": 0.07423951339442283, + "sampling/sampling_logp_difference/mean": 0.001100362363407233, + "step": 22320, + "step_time": 4.242510079868953, + "student/entropy": 0.047493799372265735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0530555764834086, + "completions/mean_terminated_length": 1.0530555764834086, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04806932732462883, + "epoch": 0.8487449208217825, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.345596313476562, + "kd/policy_loss": -1.5136056065559387, + "kd/rkl_advantage_mean": 45.77071450551351, + "kd/rkl_advantage_p95": 61.95541690190633, + "kd/rkl_advantage_std": 26.87448867658774, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5129305434246004e-07, + "loss": -6.054425048828125, + "num_tokens": 256039749.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0303898612658182, + "sampling/importance_sampling_ratio/mean": 0.9999007205168406, + "sampling/importance_sampling_ratio/min": 0.9451938609282176, + "sampling/sampling_logp_difference/max": 0.07154395127048095, + "sampling/sampling_logp_difference/mean": 0.0011915217153728008, + "step": 22350, + "step_time": 4.266965896496549, + "student/entropy": 0.04806932732462883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0497222463289897, + "completions/mean_terminated_length": 1.0497222463289897, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05095546614999572, + "epoch": 0.8498841757490602, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.980562210083008, + "kd/policy_loss": -0.8285030603408814, + "kd/rkl_advantage_mean": 47.10423259735107, + "kd/rkl_advantage_p95": 67.53190631866455, + "kd/rkl_advantage_std": 31.52599678039551, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5015379941518246e-07, + "loss": -3.314011637369792, + "num_tokens": 256345184.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0373193542162578, + "sampling/importance_sampling_ratio/mean": 1.0000489910443624, + "sampling/importance_sampling_ratio/min": 0.9590373158454895, + "sampling/sampling_logp_difference/max": 0.07277459458758434, + "sampling/sampling_logp_difference/mean": 0.0010637178279769917, + "step": 22380, + "step_time": 4.256441341298826, + "student/entropy": 0.05095546614999572 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.866666666666667, + "completions/max_terminated_length": 8.866666666666667, + "completions/mean_length": 1.1150000413258871, + "completions/mean_terminated_length": 1.1150000413258871, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05357264264797171, + "epoch": 0.8510234306763377, + "eval/gt_acc_batch": 0.002222222338120143, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.270914077758789, + "kd/policy_loss": -6.401664264996847, + "kd/rkl_advantage_mean": 78.44646010398864, + "kd/rkl_advantage_p95": 96.38515816529592, + "kd/rkl_advantage_std": 32.7054988950491, + "kd/ssd_loss": 0.0, + "learning_rate": 1.490145444879049e-07, + "loss": -25.606656901041667, + "num_tokens": 256651526.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.002222222338120143, + "rewards/gt_logging_reward/std": 0.022542615234851838, + "sampling/importance_sampling_ratio/max": 1.088283904393514, + "sampling/importance_sampling_ratio/mean": 1.0004256745179494, + "sampling/importance_sampling_ratio/min": 0.9415047605832417, + "sampling/sampling_logp_difference/max": 0.12596221573185176, + "sampling/sampling_logp_difference/mean": 0.0018537787400418893, + "step": 22410, + "step_time": 4.25413656860134, + "student/entropy": 0.05357264264797171 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.7, + "completions/max_terminated_length": 7.7, + "completions/mean_length": 1.0588889122009277, + "completions/mean_terminated_length": 1.0588889122009277, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048468183570851885, + "epoch": 0.8521626856036152, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.420711517333984, + "kd/policy_loss": -2.460099728902181, + "kd/rkl_advantage_mean": 67.24479106267293, + "kd/rkl_advantage_p95": 90.5214630206426, + "kd/rkl_advantage_std": 38.9880664686362, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4787528956062735e-07, + "loss": -9.840399169921875, + "num_tokens": 256959482.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0115631222724915, + "sampling/importance_sampling_ratio/mean": 0.9998146851857503, + "sampling/importance_sampling_ratio/min": 0.9593461473782857, + "sampling/sampling_logp_difference/max": 0.04939001036497454, + "sampling/sampling_logp_difference/mean": 0.0008257688338441463, + "step": 22440, + "step_time": 4.2943636106666725, + "student/entropy": 0.048468183570851885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.633333333333334, + "completions/max_terminated_length": 4.633333333333334, + "completions/mean_length": 1.044444457689921, + "completions/mean_terminated_length": 1.044444457689921, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04620877290144563, + "epoch": 0.8533019405308928, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.014817237854004, + "kd/policy_loss": 0.566668983300527, + "kd/rkl_advantage_mean": 22.905302715301513, + "kd/rkl_advantage_p95": 33.53687245845795, + "kd/rkl_advantage_std": 16.98388779660066, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4673603463334976e-07, + "loss": 2.266676330566406, + "num_tokens": 257262266.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0278728365898133, + "sampling/importance_sampling_ratio/mean": 1.0001017610232035, + "sampling/importance_sampling_ratio/min": 0.97399400472641, + "sampling/sampling_logp_difference/max": 0.04513709649909288, + "sampling/sampling_logp_difference/mean": 0.0009120004435923571, + "step": 22470, + "step_time": 4.264264145968385, + "student/entropy": 0.04620877290144563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.366666666666666, + "completions/max_terminated_length": 6.366666666666666, + "completions/mean_length": 1.0588889161745707, + "completions/mean_terminated_length": 1.0588889161745707, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05198392520348231, + "epoch": 0.8544411954581703, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.191102027893066, + "kd/policy_loss": -1.9563934127489726, + "kd/rkl_advantage_mean": 54.10669067700704, + "kd/rkl_advantage_p95": 73.43382639884949, + "kd/rkl_advantage_std": 32.23674956858158, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4559677970607223e-07, + "loss": -7.825571695963542, + "num_tokens": 257564230.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0538244128227234, + "sampling/importance_sampling_ratio/mean": 1.0004067460695902, + "sampling/importance_sampling_ratio/min": 0.9897910575071971, + "sampling/sampling_logp_difference/max": 0.056963276288782554, + "sampling/sampling_logp_difference/mean": 0.000857272594779109, + "step": 22500, + "step_time": 4.246603241765115, + "student/entropy": 0.05198392520348231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.966666666666667, + "completions/max_terminated_length": 6.966666666666667, + "completions/mean_length": 1.058055579662323, + "completions/mean_terminated_length": 1.058055579662323, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05047916024923325, + "epoch": 0.8555804503854479, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.225735664367676, + "kd/policy_loss": -2.0186840057373048, + "kd/rkl_advantage_mean": 58.78763276735942, + "kd/rkl_advantage_p95": 78.69548993905386, + "kd/rkl_advantage_std": 34.44826225141684, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4445752477879465e-07, + "loss": -8.074735005696615, + "num_tokens": 257885927.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0351868589719138, + "sampling/importance_sampling_ratio/mean": 1.0001811385154724, + "sampling/importance_sampling_ratio/min": 0.9764366288979848, + "sampling/sampling_logp_difference/max": 0.06035238426799575, + "sampling/sampling_logp_difference/mean": 0.0010218262488100058, + "step": 22530, + "step_time": 4.392221097569563, + "student/entropy": 0.05047916024923325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.6, + "completions/max_terminated_length": 7.6, + "completions/mean_length": 1.0794444759686788, + "completions/mean_terminated_length": 1.0794444759686788, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0498253183439374, + "epoch": 0.8567197053127255, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.440689086914062, + "kd/policy_loss": -3.418010731538137, + "kd/rkl_advantage_mean": 65.04109352429708, + "kd/rkl_advantage_p95": 84.63718746503194, + "kd/rkl_advantage_std": 33.89589221775532, + "kd/ssd_loss": 0.0, + "learning_rate": 1.433182698515171e-07, + "loss": -13.67204081217448, + "num_tokens": 258203477.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.05720511674881, + "sampling/importance_sampling_ratio/mean": 1.0002447187900543, + "sampling/importance_sampling_ratio/min": 0.956046857436498, + "sampling/sampling_logp_difference/max": 0.08189877509139479, + "sampling/sampling_logp_difference/mean": 0.0013217160982700686, + "step": 22560, + "step_time": 4.294128730034087, + "student/entropy": 0.0498253183439374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.433333333333334, + "completions/max_terminated_length": 6.433333333333334, + "completions/mean_length": 1.0588889122009277, + "completions/mean_terminated_length": 1.0588889122009277, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05058880479385455, + "epoch": 0.8578589602400031, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.65389633178711, + "kd/policy_loss": -1.2697174628575643, + "kd/rkl_advantage_mean": 49.77642674446106, + "kd/rkl_advantage_p95": 67.746400030454, + "kd/rkl_advantage_std": 30.282138077418008, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4217901492423954e-07, + "loss": -5.078871154785157, + "num_tokens": 258518097.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0178635160128275, + "sampling/importance_sampling_ratio/mean": 0.9996497770150502, + "sampling/importance_sampling_ratio/min": 0.93946905930837, + "sampling/sampling_logp_difference/max": 0.07250655769991378, + "sampling/sampling_logp_difference/mean": 0.00107601680647349, + "step": 22590, + "step_time": 4.30724921496682, + "student/entropy": 0.05058880479385455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.633333333333334, + "completions/max_terminated_length": 7.633333333333334, + "completions/mean_length": 1.070555583635966, + "completions/mean_terminated_length": 1.070555583635966, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049693181676169235, + "epoch": 0.8589982151672806, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.641942024230957, + "kd/policy_loss": -3.7528364737828572, + "kd/rkl_advantage_mean": 65.70977705319723, + "kd/rkl_advantage_p95": 84.74738891919453, + "kd/rkl_advantage_std": 33.254140705863634, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4103975999696198e-07, + "loss": -15.011347452799479, + "num_tokens": 258832879.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0213558634122213, + "sampling/importance_sampling_ratio/mean": 0.9998963375886282, + "sampling/importance_sampling_ratio/min": 0.9591644664605459, + "sampling/sampling_logp_difference/max": 0.0576068058454742, + "sampling/sampling_logp_difference/mean": 0.0010445202991832049, + "step": 22620, + "step_time": 4.221204501799366, + "student/entropy": 0.049693181676169235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.333333333333333, + "completions/max_terminated_length": 6.333333333333333, + "completions/mean_length": 1.0577777981758119, + "completions/mean_terminated_length": 1.0577777981758119, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04573160853857795, + "epoch": 0.8601374700945582, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.65601921081543, + "kd/policy_loss": -1.5788583000500997, + "kd/rkl_advantage_mean": 50.12240228652954, + "kd/rkl_advantage_p95": 67.64405285517374, + "kd/rkl_advantage_std": 29.744472971558572, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3990050506968443e-07, + "loss": -6.315433756510417, + "num_tokens": 259145647.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0282824476559957, + "sampling/importance_sampling_ratio/mean": 0.9997807164986928, + "sampling/importance_sampling_ratio/min": 0.9494515558083853, + "sampling/sampling_logp_difference/max": 0.06433708455103139, + "sampling/sampling_logp_difference/mean": 0.0010855955736284765, + "step": 22650, + "step_time": 4.312251275068653, + "student/entropy": 0.04573160853857795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0805555780728657, + "completions/mean_terminated_length": 1.0805555780728657, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04821694058676561, + "epoch": 0.8612767250218357, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 37.8263053894043, + "kd/policy_loss": -2.904249346256256, + "kd/rkl_advantage_mean": 56.348553371429446, + "kd/rkl_advantage_p95": 73.14061588446299, + "kd/rkl_advantage_std": 29.046787345409392, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3876125014240687e-07, + "loss": -11.616998291015625, + "num_tokens": 259441969.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.020039983590444, + "sampling/importance_sampling_ratio/mean": 0.9998875916004181, + "sampling/importance_sampling_ratio/min": 0.9564354995886485, + "sampling/sampling_logp_difference/max": 0.07579235931237539, + "sampling/sampling_logp_difference/mean": 0.0013504182924710525, + "step": 22680, + "step_time": 4.1631701632334925, + "student/entropy": 0.04821694058676561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.533333333333333, + "completions/max_terminated_length": 7.533333333333333, + "completions/mean_length": 1.0725000301996868, + "completions/mean_terminated_length": 1.0725000301996868, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0484843279235065, + "epoch": 0.8624159799491132, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.359246253967285, + "kd/policy_loss": -3.0489860276381173, + "kd/rkl_advantage_mean": 67.29927633603414, + "kd/rkl_advantage_p95": 90.50802722771962, + "kd/rkl_advantage_std": 38.03209475874901, + "kd/ssd_loss": 0.0, + "learning_rate": 1.376219952151293e-07, + "loss": -12.19594217936198, + "num_tokens": 259748046.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0322796066602071, + "sampling/importance_sampling_ratio/mean": 1.0000739653905233, + "sampling/importance_sampling_ratio/min": 0.966114870707194, + "sampling/sampling_logp_difference/max": 0.06333879448163013, + "sampling/sampling_logp_difference/mean": 0.0009310707183127913, + "step": 22710, + "step_time": 4.292666401234843, + "student/entropy": 0.0484843279235065 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.066666666666666, + "completions/max_terminated_length": 7.066666666666666, + "completions/mean_length": 1.0647222479184468, + "completions/mean_terminated_length": 1.0647222479184468, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048068467838068805, + "epoch": 0.8635552348763909, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 44.01982498168945, + "kd/policy_loss": -3.0538944641749066, + "kd/rkl_advantage_mean": 60.32127617200216, + "kd/rkl_advantage_p95": 78.9192290465037, + "kd/rkl_advantage_std": 32.02537170648575, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3648274028785173e-07, + "loss": -12.215579223632812, + "num_tokens": 260052039.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0281880935033163, + "sampling/importance_sampling_ratio/mean": 0.9998569428920746, + "sampling/importance_sampling_ratio/min": 0.9503328204154968, + "sampling/sampling_logp_difference/max": 0.0642913599886621, + "sampling/sampling_logp_difference/mean": 0.0010763190264697186, + "step": 22740, + "step_time": 4.249487598869504, + "student/entropy": 0.048068467838068805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.833333333333333, + "completions/max_terminated_length": 6.833333333333333, + "completions/mean_length": 1.0647222479184468, + "completions/mean_terminated_length": 1.0647222479184468, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.045379196759313345, + "epoch": 0.8646944898036684, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.965341567993164, + "kd/policy_loss": -2.2870739301045737, + "kd/rkl_advantage_mean": 58.77370971043904, + "kd/rkl_advantage_p95": 78.98776457309722, + "kd/rkl_advantage_std": 34.1051321208477, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3534348536057418e-07, + "loss": -9.148295084635416, + "num_tokens": 260348104.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0304649313290914, + "sampling/importance_sampling_ratio/mean": 0.9999698996543884, + "sampling/importance_sampling_ratio/min": 0.9627095441023509, + "sampling/sampling_logp_difference/max": 0.05704609529736141, + "sampling/sampling_logp_difference/mean": 0.0009591865683129679, + "step": 22770, + "step_time": 4.242231515665965, + "student/entropy": 0.045379196759313345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.0613889177640279, + "completions/mean_terminated_length": 1.0613889177640279, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04439775136609872, + "epoch": 0.865833744730946, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.87734603881836, + "kd/policy_loss": -1.6041294614473978, + "kd/rkl_advantage_mean": 50.48995461463928, + "kd/rkl_advantage_p95": 68.08948349952698, + "kd/rkl_advantage_std": 29.418725607792535, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3420423043329662e-07, + "loss": -6.4165196736653645, + "num_tokens": 260658229.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.017362689971924, + "sampling/importance_sampling_ratio/mean": 0.9999132454395294, + "sampling/importance_sampling_ratio/min": 0.9701435049374898, + "sampling/sampling_logp_difference/max": 0.055274537826577826, + "sampling/sampling_logp_difference/mean": 0.0009384219854837283, + "step": 22800, + "step_time": 4.338801944366908, + "student/entropy": 0.04439775136609872 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.466666666666667, + "completions/max_terminated_length": 6.466666666666667, + "completions/mean_length": 1.0688889145851135, + "completions/mean_terminated_length": 1.0688889145851135, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04648112189024687, + "epoch": 0.8669729996582235, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.466082572937012, + "kd/policy_loss": -1.1540369431177775, + "kd/rkl_advantage_mean": 49.61025698979696, + "kd/rkl_advantage_p95": 67.91421026388804, + "kd/rkl_advantage_std": 30.522223218282065, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3306497550601907e-07, + "loss": -4.616148885091146, + "num_tokens": 260962365.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0471671581268311, + "sampling/importance_sampling_ratio/mean": 1.0002856791019439, + "sampling/importance_sampling_ratio/min": 0.9690733651320139, + "sampling/sampling_logp_difference/max": 0.05557153834961355, + "sampling/sampling_logp_difference/mean": 0.0009555888658117814, + "step": 22830, + "step_time": 4.292778279931614, + "student/entropy": 0.04648112189024687 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.0, + "completions/max_terminated_length": 5.0, + "completions/mean_length": 1.0363889018694559, + "completions/mean_terminated_length": 1.0363889018694559, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04853010404234131, + "epoch": 0.8681122545855011, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.457082748413086, + "kd/policy_loss": 0.6451038678487142, + "kd/rkl_advantage_mean": 30.478362369537354, + "kd/rkl_advantage_p95": 44.74825507799785, + "kd/rkl_advantage_std": 22.697873194515704, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3192572057874148e-07, + "loss": 2.5804148356119794, + "num_tokens": 261269152.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.017084550857544, + "sampling/importance_sampling_ratio/mean": 0.999900100628535, + "sampling/importance_sampling_ratio/min": 0.9683888971805572, + "sampling/sampling_logp_difference/max": 0.04129836857318878, + "sampling/sampling_logp_difference/mean": 0.000725919990024219, + "step": 22860, + "step_time": 4.292465297200154, + "student/entropy": 0.04853010404234131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.6, + "completions/max_terminated_length": 7.6, + "completions/mean_length": 1.0738889137903849, + "completions/mean_terminated_length": 1.0738889137903849, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052559595399846636, + "epoch": 0.8692515095127786, + "eval/gt_acc_batch": 0.002222222338120143, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.664289474487305, + "kd/policy_loss": -2.714233195781708, + "kd/rkl_advantage_mean": 64.30003299713135, + "kd/rkl_advantage_p95": 84.90949822266897, + "kd/rkl_advantage_std": 35.38053793311119, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3078646565146393e-07, + "loss": -10.85693359375, + "num_tokens": 261580578.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.002222222338120143, + "rewards/gt_logging_reward/std": 0.022542615234851838, + "sampling/importance_sampling_ratio/max": 1.031674575805664, + "sampling/importance_sampling_ratio/mean": 1.0000564952691395, + "sampling/importance_sampling_ratio/min": 0.966031688451767, + "sampling/sampling_logp_difference/max": 0.06846282876407107, + "sampling/sampling_logp_difference/mean": 0.0010526816418860107, + "step": 22890, + "step_time": 4.3426795564029215, + "student/entropy": 0.052559595399846636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.433333333333334, + "completions/max_terminated_length": 7.433333333333334, + "completions/mean_length": 1.0811111450195312, + "completions/mean_terminated_length": 1.0811111450195312, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052902671756843725, + "epoch": 0.8703907644400563, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 42.2191162109375, + "kd/policy_loss": -3.7426597476005554, + "kd/rkl_advantage_mean": 65.5910496711731, + "kd/rkl_advantage_p95": 85.15855503082275, + "kd/rkl_advantage_std": 33.721583210428555, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2964721072418637e-07, + "loss": -14.970640055338542, + "num_tokens": 261896166.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0319145798683167, + "sampling/importance_sampling_ratio/mean": 0.9998296538988749, + "sampling/importance_sampling_ratio/min": 0.9466291189193725, + "sampling/sampling_logp_difference/max": 0.06030877566275497, + "sampling/sampling_logp_difference/mean": 0.0011307867796858772, + "step": 22920, + "step_time": 4.321376058936584, + "student/entropy": 0.052902671756843725 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.633333333333333, + "completions/max_terminated_length": 8.633333333333333, + "completions/mean_length": 1.084166697661082, + "completions/mean_terminated_length": 1.084166697661082, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048641862751295166, + "epoch": 0.8715300193673338, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 34.873313903808594, + "kd/policy_loss": -4.170482424894969, + "kd/rkl_advantage_mean": 78.25136453310648, + "kd/rkl_advantage_p95": 101.45137246449788, + "kd/rkl_advantage_std": 40.90877255996068, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2850795579690882e-07, + "loss": -16.681929524739584, + "num_tokens": 262209461.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0657333135604858, + "sampling/importance_sampling_ratio/mean": 1.0001387298107147, + "sampling/importance_sampling_ratio/min": 0.943292032678922, + "sampling/sampling_logp_difference/max": 0.09589460186349849, + "sampling/sampling_logp_difference/mean": 0.0013212055928306654, + "step": 22950, + "step_time": 4.333232933698067, + "student/entropy": 0.048641862751295166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.633333333333334, + "completions/max_terminated_length": 5.633333333333334, + "completions/mean_length": 1.0438889106114706, + "completions/mean_terminated_length": 1.0438889106114706, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04271627866352598, + "epoch": 0.8726692742946113, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.95979118347168, + "kd/policy_loss": -0.7529721120993297, + "kd/rkl_advantage_mean": 42.739472246170045, + "kd/rkl_advantage_p95": 61.617623043060306, + "kd/rkl_advantage_std": 28.63307475646337, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2736870086963126e-07, + "loss": -3.0118871053059895, + "num_tokens": 262509075.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0401669184366862, + "sampling/importance_sampling_ratio/mean": 1.0002121090888978, + "sampling/importance_sampling_ratio/min": 0.9800129691759746, + "sampling/sampling_logp_difference/max": 0.05439758844052752, + "sampling/sampling_logp_difference/mean": 0.0008504680372425355, + "step": 22980, + "step_time": 4.205817505665861, + "student/entropy": 0.04271627866352598 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.766666666666667, + "completions/max_terminated_length": 6.766666666666667, + "completions/mean_length": 1.0588889161745707, + "completions/mean_terminated_length": 1.0588889161745707, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04219537830601136, + "epoch": 0.8738085292218889, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.268526077270508, + "kd/policy_loss": -2.018853537241618, + "kd/rkl_advantage_mean": 57.79885279337565, + "kd/rkl_advantage_p95": 79.30571132500967, + "kd/rkl_advantage_std": 34.71263534228007, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2622944594235368e-07, + "loss": -8.075414530436198, + "num_tokens": 262816375.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.030878218015035, + "sampling/importance_sampling_ratio/mean": 1.0000466624895732, + "sampling/importance_sampling_ratio/min": 0.9650298496087392, + "sampling/sampling_logp_difference/max": 0.07369830128736794, + "sampling/sampling_logp_difference/mean": 0.0009551051451126113, + "step": 23010, + "step_time": 4.253655942199112, + "student/entropy": 0.04219537830601136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.0, + "completions/max_terminated_length": 7.0, + "completions/mean_length": 1.0722222487131754, + "completions/mean_terminated_length": 1.0722222487131754, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.046163049153983594, + "epoch": 0.8749477841491664, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.12331771850586, + "kd/policy_loss": -3.0339943687121074, + "kd/rkl_advantage_mean": 60.47276201248169, + "kd/rkl_advantage_p95": 79.43517444133758, + "kd/rkl_advantage_std": 32.511760892470676, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2509019101507615e-07, + "loss": -12.135980224609375, + "num_tokens": 263116323.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0521676222483316, + "sampling/importance_sampling_ratio/mean": 1.000182561079661, + "sampling/importance_sampling_ratio/min": 0.957440988222758, + "sampling/sampling_logp_difference/max": 0.07199168944110473, + "sampling/sampling_logp_difference/mean": 0.0012780819359856346, + "step": 23040, + "step_time": 4.2700805459656594, + "student/entropy": 0.046163049153983594 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.766666666666667, + "completions/max_terminated_length": 4.766666666666667, + "completions/mean_length": 1.0402777910232544, + "completions/mean_terminated_length": 1.0402777910232544, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.043460459696749845, + "epoch": 0.876087039076444, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 32.55610275268555, + "kd/policy_loss": 0.9426348964373271, + "kd/rkl_advantage_mean": 26.288399918874106, + "kd/rkl_advantage_p95": 39.45854512850443, + "kd/rkl_advantage_std": 20.99295804699262, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2395093608779857e-07, + "loss": 3.770539093017578, + "num_tokens": 263421500.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0266042987505595, + "sampling/importance_sampling_ratio/mean": 1.0001486857732138, + "sampling/importance_sampling_ratio/min": 0.9819967369238536, + "sampling/sampling_logp_difference/max": 0.03541671455216904, + "sampling/sampling_logp_difference/mean": 0.0006915091119784241, + "step": 23070, + "step_time": 4.226235020864988, + "student/entropy": 0.043460459696749845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.9, + "completions/max_terminated_length": 6.9, + "completions/mean_length": 1.0744444688161214, + "completions/mean_terminated_length": 1.0744444688161214, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.051997418608516456, + "epoch": 0.8772262940037215, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.824525833129883, + "kd/policy_loss": -2.9135379354159037, + "kd/rkl_advantage_mean": 56.604124784469604, + "kd/rkl_advantage_p95": 73.50959091981252, + "kd/rkl_advantage_std": 29.54892735083898, + "kd/ssd_loss": 0.0, + "learning_rate": 1.22811681160521e-07, + "loss": -11.654153442382812, + "num_tokens": 263733360.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0459978143374125, + "sampling/importance_sampling_ratio/mean": 1.000026492277781, + "sampling/importance_sampling_ratio/min": 0.9584260642528534, + "sampling/sampling_logp_difference/max": 0.07646372299641371, + "sampling/sampling_logp_difference/mean": 0.0011647108030350259, + "step": 23100, + "step_time": 4.348610141999476, + "student/entropy": 0.051997418608516456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 24.6, + "completions/max_terminated_length": 7.566666666666666, + "completions/mean_length": 1.211666707197825, + "completions/mean_terminated_length": 1.0697222510973612, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04921137765049934, + "epoch": 0.8783655489309992, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.465677261352539, + "kd/policy_loss": -2.6139034628868103, + "kd/rkl_advantage_mean": 72.21609400113424, + "kd/rkl_advantage_p95": 95.7880405664444, + "kd/rkl_advantage_std": 39.84549699028333, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2167242623324346e-07, + "loss": -10.455615234375, + "num_tokens": 264035866.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0638570189476013, + "sampling/importance_sampling_ratio/mean": 1.0000751594702402, + "sampling/importance_sampling_ratio/min": 0.954166583220164, + "sampling/sampling_logp_difference/max": 0.11175794677498439, + "sampling/sampling_logp_difference/mean": 0.0013738903304329141, + "step": 23130, + "step_time": 4.383786508302243, + "student/entropy": 0.04921137765049934 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.1, + "completions/max_terminated_length": 5.1, + "completions/mean_length": 1.0475000182787577, + "completions/mean_terminated_length": 1.0475000182787577, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04792601655547817, + "epoch": 0.8795048038582767, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 44.96208953857422, + "kd/policy_loss": 0.02605745792388916, + "kd/rkl_advantage_mean": 28.04074091911316, + "kd/rkl_advantage_p95": 39.56611484686534, + "kd/rkl_advantage_std": 18.943825447559355, + "kd/ssd_loss": 0.0, + "learning_rate": 1.205331713059659e-07, + "loss": 0.10422922770182291, + "num_tokens": 264341221.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0773919939994812, + "sampling/importance_sampling_ratio/mean": 1.000484643379847, + "sampling/importance_sampling_ratio/min": 0.9833798766136169, + "sampling/sampling_logp_difference/max": 0.05871240066674848, + "sampling/sampling_logp_difference/mean": 0.0011164629346846293, + "step": 23160, + "step_time": 4.281522153832096, + "student/entropy": 0.04792601655547817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.4, + "completions/max_terminated_length": 8.4, + "completions/mean_length": 1.0850000301996867, + "completions/mean_terminated_length": 1.0850000301996867, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05370048445959886, + "epoch": 0.8806440587855543, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 42.523616790771484, + "kd/policy_loss": -4.458071712652842, + "kd/rkl_advantage_mean": 74.50776357650757, + "kd/rkl_advantage_p95": 95.61741168498993, + "kd/rkl_advantage_std": 37.07083857357502, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1939391637868832e-07, + "loss": -17.832285563151043, + "num_tokens": 264642751.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.043540898958842, + "sampling/importance_sampling_ratio/mean": 0.9998230040073395, + "sampling/importance_sampling_ratio/min": 0.9302209218343099, + "sampling/sampling_logp_difference/max": 0.1088811205389599, + "sampling/sampling_logp_difference/mean": 0.0015123541651216024, + "step": 23190, + "step_time": 4.2377429755647125, + "student/entropy": 0.05370048445959886 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.666666666666667, + "completions/max_terminated_length": 5.666666666666667, + "completions/mean_length": 1.0519444664319357, + "completions/mean_terminated_length": 1.0519444664319357, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04971402961139878, + "epoch": 0.8817833137128318, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 39.380271911621094, + "kd/policy_loss": -0.8037902077039083, + "kd/rkl_advantage_mean": 40.901549243927, + "kd/rkl_advantage_p95": 56.834669987360634, + "kd/rkl_advantage_std": 25.80511120359103, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1825466145141077e-07, + "loss": -3.215160115559896, + "num_tokens": 264954146.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0409826199213663, + "sampling/importance_sampling_ratio/mean": 1.0001201808452607, + "sampling/importance_sampling_ratio/min": 0.9712258219718933, + "sampling/sampling_logp_difference/max": 0.05634182775393128, + "sampling/sampling_logp_difference/mean": 0.0009311818934899445, + "step": 23220, + "step_time": 4.244793160900008, + "student/entropy": 0.04971402961139878 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.666666666666667, + "completions/max_terminated_length": 7.666666666666667, + "completions/mean_length": 1.0686111410458883, + "completions/mean_terminated_length": 1.0686111410458883, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05027224893371264, + "epoch": 0.8829225686401093, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.409290313720703, + "kd/policy_loss": -3.1258043607076007, + "kd/rkl_advantage_mean": 68.22447867393494, + "kd/rkl_advantage_p95": 90.61294761498769, + "kd/rkl_advantage_std": 37.62143927812576, + "kd/ssd_loss": 0.0, + "learning_rate": 1.171154065241332e-07, + "loss": -12.50321756998698, + "num_tokens": 265256409.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0592017928759256, + "sampling/importance_sampling_ratio/mean": 1.0002704083919525, + "sampling/importance_sampling_ratio/min": 0.975405873854955, + "sampling/sampling_logp_difference/max": 0.07067473960729936, + "sampling/sampling_logp_difference/mean": 0.0009567524345281223, + "step": 23250, + "step_time": 4.255450641298861, + "student/entropy": 0.05027224893371264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.366666666666666, + "completions/max_terminated_length": 6.366666666666666, + "completions/mean_length": 1.0519444704055787, + "completions/mean_terminated_length": 1.0519444704055787, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.046300560949991146, + "epoch": 0.8840618235673869, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.63174819946289, + "kd/policy_loss": -1.6229843894640605, + "kd/rkl_advantage_mean": 53.49029429753622, + "kd/rkl_advantage_p95": 73.44847568670909, + "kd/rkl_advantage_std": 32.98397767941157, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1597615159685566e-07, + "loss": -6.491936747233073, + "num_tokens": 265571212.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0111270904541017, + "sampling/importance_sampling_ratio/mean": 0.9998190263907115, + "sampling/importance_sampling_ratio/min": 0.9607467432816823, + "sampling/sampling_logp_difference/max": 0.04708225519085924, + "sampling/sampling_logp_difference/mean": 0.0007744186708199171, + "step": 23280, + "step_time": 4.289507147667367, + "student/entropy": 0.046300560949991146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.166666666666667, + "completions/max_terminated_length": 7.166666666666667, + "completions/mean_length": 1.0608333627382913, + "completions/mean_terminated_length": 1.0608333627382913, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04841665429994464, + "epoch": 0.8852010784946645, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.865693092346191, + "kd/policy_loss": -2.8124209543069205, + "kd/rkl_advantage_mean": 62.1952704111735, + "kd/rkl_advantage_p95": 84.51066118876139, + "kd/rkl_advantage_std": 35.771706061065196, + "kd/ssd_loss": 0.0, + "learning_rate": 1.148368966695781e-07, + "loss": -11.249683634440105, + "num_tokens": 265882751.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0848162730534872, + "sampling/importance_sampling_ratio/mean": 1.0005130171775818, + "sampling/importance_sampling_ratio/min": 0.9717709620793661, + "sampling/sampling_logp_difference/max": 0.07776318808707099, + "sampling/sampling_logp_difference/mean": 0.0010184027123614214, + "step": 23310, + "step_time": 4.372311035634387, + "student/entropy": 0.04841665429994464 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.9, + "completions/max_terminated_length": 7.9, + "completions/mean_length": 1.0816666960716248, + "completions/mean_terminated_length": 1.0816666960716248, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04825698360800743, + "epoch": 0.8863403334219421, + "eval/gt_acc_batch": 0.0022222223691642285, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.841744422912598, + "kd/policy_loss": -2.2878310759862264, + "kd/rkl_advantage_mean": 59.73633699417114, + "kd/rkl_advantage_p95": 79.33637948830922, + "kd/rkl_advantage_std": 33.20884027481079, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1369764174230052e-07, + "loss": -9.15132548014323, + "num_tokens": 266195125.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0022222223070760567, + "rewards/gt_logging_reward/std": 0.020440502216418584, + "sampling/importance_sampling_ratio/max": 1.0687661170959473, + "sampling/importance_sampling_ratio/mean": 1.000311056772868, + "sampling/importance_sampling_ratio/min": 0.9522455732027689, + "sampling/sampling_logp_difference/max": 0.10390482475049793, + "sampling/sampling_logp_difference/mean": 0.0016250604918847482, + "step": 23340, + "step_time": 4.427989940296296, + "student/entropy": 0.04825698360800743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.8, + "completions/max_terminated_length": 7.8, + "completions/mean_length": 1.0811111410458882, + "completions/mean_terminated_length": 1.0811111410458882, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0458503264002502, + "epoch": 0.8874795883492196, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.742204666137695, + "kd/policy_loss": -4.751358326276144, + "kd/rkl_advantage_mean": 71.11314250628153, + "kd/rkl_advantage_p95": 90.557062180837, + "kd/rkl_advantage_std": 33.7577063113451, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1255838681502297e-07, + "loss": -19.005434163411458, + "num_tokens": 266509913.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0222576061884563, + "sampling/importance_sampling_ratio/mean": 0.9997627516587575, + "sampling/importance_sampling_ratio/min": 0.9493596076965332, + "sampling/sampling_logp_difference/max": 0.062581863362963, + "sampling/sampling_logp_difference/mean": 0.001073482803379496, + "step": 23370, + "step_time": 4.430698609163422, + "student/entropy": 0.0458503264002502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0502778013547263, + "completions/mean_terminated_length": 1.0502778013547263, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04398968458796541, + "epoch": 0.8886188432764972, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.790007591247559, + "kd/policy_loss": -1.159615397453308, + "kd/rkl_advantage_mean": 47.480741580327354, + "kd/rkl_advantage_p95": 67.90184641679129, + "kd/rkl_advantage_std": 30.816815066337586, + "kd/ssd_loss": 0.0, + "learning_rate": 1.114191318877454e-07, + "loss": -4.638461303710938, + "num_tokens": 266817478.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.045313592751821, + "sampling/importance_sampling_ratio/mean": 1.0002126455307008, + "sampling/importance_sampling_ratio/min": 0.9763972183068593, + "sampling/sampling_logp_difference/max": 0.05467496269072095, + "sampling/sampling_logp_difference/mean": 0.0008869823926943354, + "step": 23400, + "step_time": 4.349784488533138, + "student/entropy": 0.04398968458796541 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.0588889161745707, + "completions/mean_terminated_length": 1.0588889161745707, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.043687812983989716, + "epoch": 0.8897580982037747, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.292054176330566, + "kd/policy_loss": -1.5941163420677185, + "kd/rkl_advantage_mean": 52.795183801651, + "kd/rkl_advantage_p95": 73.24558191299438, + "kd/rkl_advantage_std": 32.870678221186004, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1027987696046786e-07, + "loss": -6.37646484375, + "num_tokens": 267129994.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0509464661280314, + "sampling/importance_sampling_ratio/mean": 1.0000505924224854, + "sampling/importance_sampling_ratio/min": 0.9557019253571828, + "sampling/sampling_logp_difference/max": 0.07926164274103939, + "sampling/sampling_logp_difference/mean": 0.0010989763167647956, + "step": 23430, + "step_time": 4.303260972167966, + "student/entropy": 0.043687812983989716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.7, + "completions/max_terminated_length": 6.7, + "completions/mean_length": 1.063611137866974, + "completions/mean_terminated_length": 1.063611137866974, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.045399216221024594, + "epoch": 0.8908973531310523, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.207420349121094, + "kd/policy_loss": -1.6343098640441895, + "kd/rkl_advantage_mean": 53.469484376907346, + "kd/rkl_advantage_p95": 73.17413905461629, + "kd/rkl_advantage_std": 32.2827671011289, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0914062203319029e-07, + "loss": -6.537238566080729, + "num_tokens": 267440991.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.022946306069692, + "sampling/importance_sampling_ratio/mean": 0.9996450006961822, + "sampling/importance_sampling_ratio/min": 0.9458068907260895, + "sampling/sampling_logp_difference/max": 0.07411331110633909, + "sampling/sampling_logp_difference/mean": 0.0010288503263533736, + "step": 23460, + "step_time": 4.295683391168131, + "student/entropy": 0.045399216221024594 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.633333333333334, + "completions/max_terminated_length": 5.633333333333334, + "completions/mean_length": 1.0516666849454244, + "completions/mean_terminated_length": 1.0516666849454244, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.045025383774191144, + "epoch": 0.8920366080583298, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.90581512451172, + "kd/policy_loss": -1.125226374467214, + "kd/rkl_advantage_mean": 41.72066642443339, + "kd/rkl_advantage_p95": 56.35324037075043, + "kd/rkl_advantage_std": 24.960581133762997, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0800136710591273e-07, + "loss": -4.500905354817708, + "num_tokens": 267751921.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0191421469052633, + "sampling/importance_sampling_ratio/mean": 0.9999614238739014, + "sampling/importance_sampling_ratio/min": 0.9731647710005442, + "sampling/sampling_logp_difference/max": 0.04388265850332876, + "sampling/sampling_logp_difference/mean": 0.0006902239275708174, + "step": 23490, + "step_time": 4.299250910466071, + "student/entropy": 0.045025383774191144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.866666666666666, + "completions/max_terminated_length": 6.866666666666666, + "completions/mean_length": 1.0666666865348815, + "completions/mean_terminated_length": 1.0666666865348815, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049532569199800494, + "epoch": 0.8931758629856074, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.086088180541992, + "kd/policy_loss": -2.2693570494651794, + "kd/rkl_advantage_mean": 55.621128511428836, + "kd/rkl_advantage_p95": 73.64331755638122, + "kd/rkl_advantage_std": 30.694108781218528, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0686211217863516e-07, + "loss": -9.077427164713542, + "num_tokens": 268058841.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0212403257687888, + "sampling/importance_sampling_ratio/mean": 0.9998301823933919, + "sampling/importance_sampling_ratio/min": 0.9574031829833984, + "sampling/sampling_logp_difference/max": 0.07531709295387069, + "sampling/sampling_logp_difference/mean": 0.001285246849389902, + "step": 23520, + "step_time": 4.2725679773007865, + "student/entropy": 0.049532569199800494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.333333333333333, + "completions/max_terminated_length": 6.333333333333333, + "completions/mean_length": 1.0625000238418578, + "completions/mean_terminated_length": 1.0625000238418578, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04811475913350781, + "epoch": 0.894315117912885, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.619202613830566, + "kd/policy_loss": -2.179751288890839, + "kd/rkl_advantage_mean": 50.46044583320618, + "kd/rkl_advantage_p95": 67.52014046510061, + "kd/rkl_advantage_std": 28.306338477134705, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0572285725135762e-07, + "loss": -8.719005330403645, + "num_tokens": 268370370.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.013440457979838, + "sampling/importance_sampling_ratio/mean": 0.9995799322923025, + "sampling/importance_sampling_ratio/min": 0.9520804842313131, + "sampling/sampling_logp_difference/max": 0.054388152156025174, + "sampling/sampling_logp_difference/mean": 0.001008023219765164, + "step": 23550, + "step_time": 4.272141078900313, + "student/entropy": 0.04811475913350781 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.766666666666667, + "completions/max_terminated_length": 8.766666666666667, + "completions/mean_length": 1.0963889201482138, + "completions/mean_terminated_length": 1.0963889201482138, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04765630674858888, + "epoch": 0.8954543728401625, + "eval/gt_acc_batch": 0.0025000001303851606, + "frac_reward_zero_std": 1.0, + "grad_norm": 249.10205078125, + "kd/policy_loss": -6.09116059144338, + "kd/rkl_advantage_mean": 84.43277961413065, + "kd/rkl_advantage_p95": 107.47865318457285, + "kd/rkl_advantage_std": 39.31902610162894, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0458360232408005e-07, + "loss": -24.364644368489582, + "num_tokens": 268675653.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000001303851606, + "rewards/gt_logging_reward/std": 0.025585518529017765, + "sampling/importance_sampling_ratio/max": 1.0383544524510702, + "sampling/importance_sampling_ratio/mean": 0.999712719519933, + "sampling/importance_sampling_ratio/min": 0.9331427137056987, + "sampling/sampling_logp_difference/max": 0.09855003752745688, + "sampling/sampling_logp_difference/mean": 0.0014556094713043422, + "step": 23580, + "step_time": 4.324026855265644, + "student/entropy": 0.04765630674858888 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.4, + "completions/max_terminated_length": 8.4, + "completions/mean_length": 1.078611139456431, + "completions/mean_terminated_length": 1.078611139456431, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04283688412979245, + "epoch": 0.8965936277674401, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 30.94443130493164, + "kd/policy_loss": -4.126782627900441, + "kd/rkl_advantage_mean": 75.28290824890136, + "kd/rkl_advantage_p95": 96.52857191562653, + "kd/rkl_advantage_std": 38.2694572865963, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0344434739680248e-07, + "loss": -16.507127888997395, + "num_tokens": 268984184.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0477675596872966, + "sampling/importance_sampling_ratio/mean": 1.0001682003339132, + "sampling/importance_sampling_ratio/min": 0.9659343421459198, + "sampling/sampling_logp_difference/max": 0.07147404806067546, + "sampling/sampling_logp_difference/mean": 0.0010127652601416533, + "step": 23610, + "step_time": 4.3852198607026365, + "student/entropy": 0.04283688412979245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.133333333333334, + "completions/max_terminated_length": 6.133333333333334, + "completions/mean_length": 1.0480555733044943, + "completions/mean_terminated_length": 1.0480555733044943, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.038333217675487204, + "epoch": 0.8977328826947176, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.04443645477295, + "kd/policy_loss": -0.9439691265424093, + "kd/rkl_advantage_mean": 45.7324935913086, + "kd/rkl_advantage_p95": 62.30723026593526, + "kd/rkl_advantage_std": 28.195930046836533, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0230509246952493e-07, + "loss": -3.7758756001790363, + "num_tokens": 269297605.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0027865449587503, + "sampling/importance_sampling_ratio/mean": 0.9997089684009552, + "sampling/importance_sampling_ratio/min": 0.9533313810825348, + "sampling/sampling_logp_difference/max": 0.048549903308351836, + "sampling/sampling_logp_difference/mean": 0.0006291368898625175, + "step": 23640, + "step_time": 4.296332406466536, + "student/entropy": 0.038333217675487204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.065833361943563, + "completions/mean_terminated_length": 1.065833361943563, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04104212345555425, + "epoch": 0.8988721376219952, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.45112419128418, + "kd/policy_loss": -2.3304541309674582, + "kd/rkl_advantage_mean": 59.14353632926941, + "kd/rkl_advantage_p95": 78.71284461816153, + "kd/rkl_advantage_std": 33.356969594955444, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0116583754224736e-07, + "loss": -9.321818033854166, + "num_tokens": 269598850.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0408131996790568, + "sampling/importance_sampling_ratio/mean": 0.9997491975625356, + "sampling/importance_sampling_ratio/min": 0.9259647925694784, + "sampling/sampling_logp_difference/max": 0.09928258676081896, + "sampling/sampling_logp_difference/mean": 0.0014069739404173258, + "step": 23670, + "step_time": 4.290059835898864, + "student/entropy": 0.04104212345555425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.166666666666667, + "completions/max_terminated_length": 6.166666666666667, + "completions/mean_length": 1.0566666960716247, + "completions/mean_terminated_length": 1.0566666960716247, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.040858944257100425, + "epoch": 0.9000113925492728, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.195083618164062, + "kd/policy_loss": -1.550406555334727, + "kd/rkl_advantage_mean": 51.56597922643026, + "kd/rkl_advantage_p95": 73.17353804111481, + "kd/rkl_advantage_std": 32.96310789386431, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0002658261496981e-07, + "loss": -6.201623026529948, + "num_tokens": 269903726.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0407552202542623, + "sampling/importance_sampling_ratio/mean": 1.0001488367716471, + "sampling/importance_sampling_ratio/min": 0.966803640127182, + "sampling/sampling_logp_difference/max": 0.07429542847288152, + "sampling/sampling_logp_difference/mean": 0.0010609343841982385, + "step": 23700, + "step_time": 4.314753751268047, + "student/entropy": 0.040858944257100425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.433333333333334, + "completions/max_terminated_length": 6.433333333333334, + "completions/mean_length": 1.0644444704055787, + "completions/mean_terminated_length": 1.0644444704055787, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.041579989312837524, + "epoch": 0.9011506474765504, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.229206085205078, + "kd/policy_loss": -2.1415175358454386, + "kd/rkl_advantage_mean": 50.49646841684977, + "kd/rkl_advantage_p95": 67.51251905759176, + "kd/rkl_advantage_std": 28.24423257311185, + "kd/ssd_loss": 0.0, + "learning_rate": 9.888732768769224e-08, + "loss": -8.566071573893229, + "num_tokens": 270208678.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0319327076276144, + "sampling/importance_sampling_ratio/mean": 0.9999551713466645, + "sampling/importance_sampling_ratio/min": 0.9647069712479909, + "sampling/sampling_logp_difference/max": 0.06510187374272694, + "sampling/sampling_logp_difference/mean": 0.00099186649749754, + "step": 23730, + "step_time": 4.282156901697939, + "student/entropy": 0.041579989312837524 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.833333333333333, + "completions/max_terminated_length": 4.833333333333333, + "completions/mean_length": 1.043333347638448, + "completions/mean_terminated_length": 1.043333347638448, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04296016879379749, + "epoch": 0.9022899024038279, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.350014686584473, + "kd/policy_loss": 0.06171230872472127, + "kd/rkl_advantage_mean": 27.819320249557496, + "kd/rkl_advantage_p95": 39.22122294902802, + "kd/rkl_advantage_std": 18.699570485949515, + "kd/ssd_loss": 0.0, + "learning_rate": 9.774807276041469e-08, + "loss": 0.24684890111287436, + "num_tokens": 270510834.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0593487819035847, + "sampling/importance_sampling_ratio/mean": 1.000369014342626, + "sampling/importance_sampling_ratio/min": 0.9783653934796651, + "sampling/sampling_logp_difference/max": 0.055768538576861225, + "sampling/sampling_logp_difference/mean": 0.0009969732724130153, + "step": 23760, + "step_time": 4.347220717368085, + "student/entropy": 0.04296016879379749 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.533333333333333, + "completions/max_terminated_length": 5.533333333333333, + "completions/mean_length": 1.0508333563804626, + "completions/mean_terminated_length": 1.0508333563804626, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04749724861855308, + "epoch": 0.9034291573311054, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.388795852661133, + "kd/policy_loss": -0.7203752438227335, + "kd/rkl_advantage_mean": 40.19621775945028, + "kd/rkl_advantage_p95": 55.956116223335265, + "kd/rkl_advantage_std": 25.74872475961844, + "kd/ssd_loss": 0.0, + "learning_rate": 9.660881783313712e-08, + "loss": -2.8815017700195313, + "num_tokens": 270816113.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0253026763598123, + "sampling/importance_sampling_ratio/mean": 0.9998326241970062, + "sampling/importance_sampling_ratio/min": 0.95230153799057, + "sampling/sampling_logp_difference/max": 0.0570972941039751, + "sampling/sampling_logp_difference/mean": 0.0009708473672314237, + "step": 23790, + "step_time": 4.29280339916586, + "student/entropy": 0.04749724861855308 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.1, + "completions/max_terminated_length": 8.1, + "completions/mean_length": 1.0850000381469727, + "completions/mean_terminated_length": 1.0850000381469727, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049210240691900255, + "epoch": 0.904568412258383, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.385687828063965, + "kd/policy_loss": -4.508092649777731, + "kd/rkl_advantage_mean": 74.59294873873392, + "kd/rkl_advantage_p95": 96.20361250241598, + "kd/rkl_advantage_std": 37.42957333624363, + "kd/ssd_loss": 0.0, + "learning_rate": 9.546956290585956e-08, + "loss": -18.03237101236979, + "num_tokens": 271118067.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0442359646161397, + "sampling/importance_sampling_ratio/mean": 0.9998793502648672, + "sampling/importance_sampling_ratio/min": 0.9386113425095876, + "sampling/sampling_logp_difference/max": 0.1039937542906652, + "sampling/sampling_logp_difference/mean": 0.0013726390088171078, + "step": 23820, + "step_time": 4.330096539766236, + "student/entropy": 0.049210240691900255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.066666666666666, + "completions/max_terminated_length": 7.066666666666666, + "completions/mean_length": 1.0747222503026326, + "completions/mean_terminated_length": 1.0747222503026326, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04717744089042147, + "epoch": 0.9057076671856605, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.495144844055176, + "kd/policy_loss": -3.3722556392351786, + "kd/rkl_advantage_mean": 57.603861570358276, + "kd/rkl_advantage_p95": 73.51285157998403, + "kd/rkl_advantage_std": 28.273212453722955, + "kd/ssd_loss": 0.0, + "learning_rate": 9.433030797858201e-08, + "loss": -13.489019775390625, + "num_tokens": 271420472.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.019499711940685907, + "sampling/importance_sampling_ratio/max": 1.0311639547348022, + "sampling/importance_sampling_ratio/mean": 1.0000227550665537, + "sampling/importance_sampling_ratio/min": 0.9602817893028259, + "sampling/sampling_logp_difference/max": 0.05349065076249341, + "sampling/sampling_logp_difference/mean": 0.0009169567284213069, + "step": 23850, + "step_time": 4.302685132929764, + "student/entropy": 0.04717744089042147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.0, + "completions/max_terminated_length": 6.0, + "completions/mean_length": 1.0583333571751912, + "completions/mean_terminated_length": 1.0583333571751912, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04924160412823161, + "epoch": 0.9068469221129382, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.165512084960938, + "kd/policy_loss": -1.841904882589976, + "kd/rkl_advantage_mean": 46.43041737874349, + "kd/rkl_advantage_p95": 61.95964398384094, + "kd/rkl_advantage_std": 26.149573553105196, + "kd/ssd_loss": 0.0, + "learning_rate": 9.319105305130444e-08, + "loss": -7.3676198323567705, + "num_tokens": 271729594.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0290790836016337, + "sampling/importance_sampling_ratio/mean": 0.9999085962772369, + "sampling/importance_sampling_ratio/min": 0.9501220504442851, + "sampling/sampling_logp_difference/max": 0.0732690014410764, + "sampling/sampling_logp_difference/mean": 0.001038596136883522, + "step": 23880, + "step_time": 4.304293369435375, + "student/entropy": 0.04924160412823161 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.233333333333333, + "completions/max_terminated_length": 6.233333333333333, + "completions/mean_length": 1.0633333563804626, + "completions/mean_terminated_length": 1.0633333563804626, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04838614693532387, + "epoch": 0.9079861770402157, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 38.5114631652832, + "kd/policy_loss": -2.1536469062169394, + "kd/rkl_advantage_mean": 47.87482899030049, + "kd/rkl_advantage_p95": 61.8131893714269, + "kd/rkl_advantage_std": 25.108561536669733, + "kd/ssd_loss": 0.0, + "learning_rate": 9.205179812402688e-08, + "loss": -8.614589436848958, + "num_tokens": 272032086.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.0204557379086812, + "sampling/importance_sampling_ratio/mean": 1.0000152826309203, + "sampling/importance_sampling_ratio/min": 0.9750521620114644, + "sampling/sampling_logp_difference/max": 0.040565045752252143, + "sampling/sampling_logp_difference/mean": 0.000783057369699236, + "step": 23910, + "step_time": 4.28388834153302, + "student/entropy": 0.04838614693532387 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.7, + "completions/max_terminated_length": 5.7, + "completions/mean_length": 1.0502778013547263, + "completions/mean_terminated_length": 1.0502778013547263, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049211578505734606, + "epoch": 0.9091254319674933, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.502763748168945, + "kd/policy_loss": -1.1492752114931741, + "kd/rkl_advantage_mean": 41.20302963256836, + "kd/rkl_advantage_p95": 56.50728022257487, + "kd/rkl_advantage_std": 24.954479483763375, + "kd/ssd_loss": 0.0, + "learning_rate": 9.091254319674931e-08, + "loss": -4.597099812825521, + "num_tokens": 272352995.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.016296648979187, + "sampling/importance_sampling_ratio/mean": 1.0000189065933227, + "sampling/importance_sampling_ratio/min": 0.9827952881654104, + "sampling/sampling_logp_difference/max": 0.032482077429691954, + "sampling/sampling_logp_difference/mean": 0.0007117814122466371, + "step": 23940, + "step_time": 4.330478913268113, + "student/entropy": 0.049211578505734606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.3, + "completions/max_terminated_length": 7.3, + "completions/mean_length": 1.076388919353485, + "completions/mean_terminated_length": 1.076388919353485, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05195799817641576, + "epoch": 0.9102646868947708, + "eval/gt_acc_batch": 0.0025000001614292463, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.533239364624023, + "kd/policy_loss": -3.029582997163137, + "kd/rkl_advantage_mean": 63.85713580449422, + "kd/rkl_advantage_p95": 84.74161473910014, + "kd/rkl_advantage_std": 35.273025730252265, + "kd/ssd_loss": 0.0, + "learning_rate": 8.977328826947177e-08, + "loss": -12.118333943684895, + "num_tokens": 272667070.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000000993410745, + "rewards/gt_logging_reward/std": 0.021682794392108917, + "sampling/importance_sampling_ratio/max": 1.026629137992859, + "sampling/importance_sampling_ratio/mean": 0.9997331301371256, + "sampling/importance_sampling_ratio/min": 0.9498529632886251, + "sampling/sampling_logp_difference/max": 0.06393319009803236, + "sampling/sampling_logp_difference/mean": 0.0011243826748492817, + "step": 23970, + "step_time": 4.290254561362962, + "student/entropy": 0.05195799817641576 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.266666666666667, + "completions/max_terminated_length": 7.266666666666667, + "completions/mean_length": 1.0663889169692993, + "completions/mean_terminated_length": 1.0663889169692993, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0508183809928596, + "epoch": 0.9114039418220484, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.75180435180664, + "kd/policy_loss": -2.6890099624792736, + "kd/rkl_advantage_mean": 63.471430699030556, + "kd/rkl_advantage_p95": 84.68637353579203, + "kd/rkl_advantage_std": 35.75594659298658, + "kd/ssd_loss": 0.0, + "learning_rate": 8.86340333421942e-08, + "loss": -10.756039428710938, + "num_tokens": 272974085.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0574281692504883, + "sampling/importance_sampling_ratio/mean": 1.0001977145671845, + "sampling/importance_sampling_ratio/min": 0.9584513485431672, + "sampling/sampling_logp_difference/max": 0.07495251419022679, + "sampling/sampling_logp_difference/mean": 0.0012676978396484629, + "step": 24000, + "step_time": 4.3529333292351415, + "student/entropy": 0.0508183809928596 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.7, + "completions/max_terminated_length": 7.7, + "completions/mean_length": 1.0736111442248026, + "completions/mean_terminated_length": 1.0736111442248026, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04844230472420653, + "epoch": 0.9125431967493259, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 38.38010025024414, + "kd/policy_loss": -4.1319663961728414, + "kd/rkl_advantage_mean": 69.94574666023254, + "kd/rkl_advantage_p95": 90.10309161345164, + "kd/rkl_advantage_std": 35.28431395292282, + "kd/ssd_loss": 0.0, + "learning_rate": 8.749477841491663e-08, + "loss": -16.527867635091145, + "num_tokens": 273278798.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0712505062421163, + "sampling/importance_sampling_ratio/mean": 1.0002658784389495, + "sampling/importance_sampling_ratio/min": 0.953134701649348, + "sampling/sampling_logp_difference/max": 0.09639278064326694, + "sampling/sampling_logp_difference/mean": 0.0013925739816234757, + "step": 24030, + "step_time": 4.423237143367199, + "student/entropy": 0.04844230472420653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.166666666666667, + "completions/max_terminated_length": 6.166666666666667, + "completions/mean_length": 1.061666691303253, + "completions/mean_terminated_length": 1.061666691303253, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04785337643697858, + "epoch": 0.9136824516766034, + "eval/gt_acc_batch": 0.0011111112001041571, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.156811714172363, + "kd/policy_loss": -2.1365119397640226, + "kd/rkl_advantage_mean": 49.842233737309776, + "kd/rkl_advantage_p95": 67.51674472490946, + "kd/rkl_advantage_std": 28.409701419870057, + "kd/ssd_loss": 0.0, + "learning_rate": 8.635552348763908e-08, + "loss": -8.546046956380208, + "num_tokens": 273580284.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111380159855, + "rewards/gt_logging_reward/std": 0.008268889039754868, + "sampling/importance_sampling_ratio/max": 1.055009917418162, + "sampling/importance_sampling_ratio/mean": 1.0001947184403737, + "sampling/importance_sampling_ratio/min": 0.9636470258235932, + "sampling/sampling_logp_difference/max": 0.06721094379511973, + "sampling/sampling_logp_difference/mean": 0.0010575274335375676, + "step": 24060, + "step_time": 4.384511005798898, + "student/entropy": 0.04785337643697858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.366666666666666, + "completions/max_terminated_length": 7.366666666666666, + "completions/mean_length": 1.0772222518920898, + "completions/mean_terminated_length": 1.0772222518920898, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.046249040961265565, + "epoch": 0.9148217066038811, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 38.50188446044922, + "kd/policy_loss": -3.611657687028249, + "kd/rkl_advantage_mean": 61.582856019337974, + "kd/rkl_advantage_p95": 79.2584853887558, + "kd/rkl_advantage_std": 30.56291253566742, + "kd/ssd_loss": 0.0, + "learning_rate": 8.521626856036152e-08, + "loss": -14.446630859375, + "num_tokens": 273886794.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.047684113184611, + "sampling/importance_sampling_ratio/mean": 1.000194388628006, + "sampling/importance_sampling_ratio/min": 0.9611789762973786, + "sampling/sampling_logp_difference/max": 0.07679280125691244, + "sampling/sampling_logp_difference/mean": 0.0010787309322040528, + "step": 24090, + "step_time": 4.414287464837738, + "student/entropy": 0.046249040961265565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.866666666666666, + "completions/max_terminated_length": 4.866666666666666, + "completions/mean_length": 1.035277791817983, + "completions/mean_terminated_length": 1.035277791817983, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047178208672751985, + "epoch": 0.9159609615311586, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.843398094177246, + "kd/policy_loss": 0.9702996969223022, + "kd/rkl_advantage_mean": 29.543603388468423, + "kd/rkl_advantage_p95": 44.72999172210693, + "kd/rkl_advantage_std": 23.741431648035846, + "kd/ssd_loss": 0.0, + "learning_rate": 8.407701363308397e-08, + "loss": 3.8811991373697916, + "num_tokens": 274195177.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0324573596318563, + "sampling/importance_sampling_ratio/mean": 1.000251841545105, + "sampling/importance_sampling_ratio/min": 0.9944624284903208, + "sampling/sampling_logp_difference/max": 0.03010519960274299, + "sampling/sampling_logp_difference/mean": 0.0006294949989145, + "step": 24120, + "step_time": 4.477281581232576, + "student/entropy": 0.047178208672751985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.933333333333334, + "completions/max_terminated_length": 6.933333333333334, + "completions/mean_length": 1.0719444751739502, + "completions/mean_terminated_length": 1.0719444751739502, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.052002884168177846, + "epoch": 0.9171002164584362, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.827194213867188, + "kd/policy_loss": -2.613659413655599, + "kd/rkl_advantage_mean": 55.097008562088014, + "kd/rkl_advantage_p95": 73.2152223110199, + "kd/rkl_advantage_std": 30.085294116536776, + "kd/ssd_loss": 0.0, + "learning_rate": 8.29377587058064e-08, + "loss": -10.454638671875, + "num_tokens": 274492692.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.048806913693746, + "sampling/importance_sampling_ratio/mean": 1.0001303295294444, + "sampling/importance_sampling_ratio/min": 0.9488016615311304, + "sampling/sampling_logp_difference/max": 0.09903917494229972, + "sampling/sampling_logp_difference/mean": 0.0014697449320616822, + "step": 24150, + "step_time": 4.415427773935759, + "student/entropy": 0.052002884168177846 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.766666666666667, + "completions/max_terminated_length": 8.766666666666667, + "completions/mean_length": 1.1147222677866617, + "completions/mean_terminated_length": 1.1147222677866617, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0525738270021975, + "epoch": 0.9182394713857137, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 36.654884338378906, + "kd/policy_loss": -5.780306263764699, + "kd/rkl_advantage_mean": 84.6426478544871, + "kd/rkl_advantage_p95": 107.35587903658549, + "kd/rkl_advantage_std": 40.44498310685158, + "kd/ssd_loss": 0.0, + "learning_rate": 8.179850377852884e-08, + "loss": -23.12122802734375, + "num_tokens": 274808161.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0385854442914326, + "sampling/importance_sampling_ratio/mean": 0.9996355831623077, + "sampling/importance_sampling_ratio/min": 0.9275064071019491, + "sampling/sampling_logp_difference/max": 0.10440215415631732, + "sampling/sampling_logp_difference/mean": 0.001509464731013092, + "step": 24180, + "step_time": 4.517041007667528, + "student/entropy": 0.0525738270021975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.466666666666667, + "completions/max_terminated_length": 7.466666666666667, + "completions/mean_length": 1.0755555868148803, + "completions/mean_terminated_length": 1.0755555868148803, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04803983960300684, + "epoch": 0.9193787263129913, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.195714950561523, + "kd/policy_loss": -4.124913203716278, + "kd/rkl_advantage_mean": 66.46265080769857, + "kd/rkl_advantage_p95": 85.00524960358938, + "kd/rkl_advantage_std": 32.85135819017887, + "kd/ssd_loss": 0.0, + "learning_rate": 8.065924885125127e-08, + "loss": -16.49965616861979, + "num_tokens": 275122033.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0946678042411804, + "sampling/importance_sampling_ratio/mean": 1.0003726144631704, + "sampling/importance_sampling_ratio/min": 0.9479821900526683, + "sampling/sampling_logp_difference/max": 0.09907426158897578, + "sampling/sampling_logp_difference/mean": 0.0014529868572329481, + "step": 24210, + "step_time": 4.409343508165815, + "student/entropy": 0.04803983960300684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.7, + "completions/max_terminated_length": 5.7, + "completions/mean_length": 1.0422222375869752, + "completions/mean_terminated_length": 1.0422222375869752, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04474618475263317, + "epoch": 0.9205179812402688, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.795917510986328, + "kd/policy_loss": -0.20540785789489746, + "kd/rkl_advantage_mean": 40.28006199200948, + "kd/rkl_advantage_p95": 56.71903080940247, + "kd/rkl_advantage_std": 26.958952873945236, + "kd/ssd_loss": 0.0, + "learning_rate": 7.951999392397373e-08, + "loss": -0.8216310501098633, + "num_tokens": 275423889.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.011881987253825, + "sampling/importance_sampling_ratio/mean": 0.9999196191628774, + "sampling/importance_sampling_ratio/min": 0.9765690108140309, + "sampling/sampling_logp_difference/max": 0.0206815663104256, + "sampling/sampling_logp_difference/mean": 0.0005549656144770173, + "step": 24240, + "step_time": 4.429932666332752, + "student/entropy": 0.04474618475263317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.266666666666667, + "completions/max_terminated_length": 5.266666666666667, + "completions/mean_length": 1.052500021457672, + "completions/mean_terminated_length": 1.052500021457672, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04781378818054994, + "epoch": 0.9216572361675465, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.834612846374512, + "kd/policy_loss": -1.3882596731185912, + "kd/rkl_advantage_mean": 37.34750164349874, + "kd/rkl_advantage_p95": 50.52439551353454, + "kd/rkl_advantage_std": 21.414595529437065, + "kd/ssd_loss": 0.0, + "learning_rate": 7.838073899669616e-08, + "loss": -5.553039042154948, + "num_tokens": 275733974.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0609177509943644, + "sampling/importance_sampling_ratio/mean": 1.0004789292812348, + "sampling/importance_sampling_ratio/min": 0.9891688406467438, + "sampling/sampling_logp_difference/max": 0.064450190262869, + "sampling/sampling_logp_difference/mean": 0.0009482235337297122, + "step": 24270, + "step_time": 4.498605320999923, + "student/entropy": 0.04781378818054994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 22.3, + "completions/max_terminated_length": 5.6, + "completions/mean_length": 1.183333365122477, + "completions/mean_terminated_length": 1.0414122502009073, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04992074053734541, + "epoch": 0.922796491094824, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.003915786743164, + "kd/policy_loss": -0.33807428876558937, + "kd/rkl_advantage_mean": 41.29031934738159, + "kd/rkl_advantage_p95": 56.23817559083303, + "kd/rkl_advantage_std": 25.099328701694805, + "kd/ssd_loss": 0.0, + "learning_rate": 7.724148406941859e-08, + "loss": -1.3522972106933593, + "num_tokens": 276042898.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0441672166188558, + "sampling/importance_sampling_ratio/mean": 1.000046004851659, + "sampling/importance_sampling_ratio/min": 0.9557707796494166, + "sampling/sampling_logp_difference/max": 0.0718414139157782, + "sampling/sampling_logp_difference/mean": 0.0009963443085628872, + "step": 24300, + "step_time": 4.507249213164808, + "student/entropy": 0.04992074053734541 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.266666666666667, + "completions/max_terminated_length": 8.266666666666667, + "completions/mean_length": 1.0861111442248026, + "completions/mean_terminated_length": 1.0861111442248026, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05055412466948231, + "epoch": 0.9239357460221016, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.704442977905273, + "kd/policy_loss": -4.430734141667684, + "kd/rkl_advantage_mean": 75.0232611020406, + "kd/rkl_advantage_p95": 95.86363352139792, + "kd/rkl_advantage_std": 37.25457036097844, + "kd/ssd_loss": 0.0, + "learning_rate": 7.610222914214103e-08, + "loss": -17.72293701171875, + "num_tokens": 276342992.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0363068103790283, + "sampling/importance_sampling_ratio/mean": 0.9999526937802633, + "sampling/importance_sampling_ratio/min": 0.9626304507255554, + "sampling/sampling_logp_difference/max": 0.07415844726686677, + "sampling/sampling_logp_difference/mean": 0.0011960307262294616, + "step": 24330, + "step_time": 4.429521824268644, + "student/entropy": 0.05055412466948231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.0725000301996868, + "completions/mean_terminated_length": 1.0725000301996868, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04819044756392638, + "epoch": 0.9250750009493791, + "eval/gt_acc_batch": 0.002222222338120143, + "frac_reward_zero_std": 1.0, + "grad_norm": 184.55126953125, + "kd/policy_loss": -2.1485015233357747, + "kd/rkl_advantage_mean": 54.401175292332965, + "kd/rkl_advantage_p95": 73.13002263704935, + "kd/rkl_advantage_std": 31.408411008119582, + "kd/ssd_loss": 0.0, + "learning_rate": 7.496297421486348e-08, + "loss": -8.59400634765625, + "num_tokens": 276651789.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.002222222338120143, + "rewards/gt_logging_reward/std": 0.02074200411637624, + "sampling/importance_sampling_ratio/max": 1.0355929176012675, + "sampling/importance_sampling_ratio/mean": 0.9999695499738057, + "sampling/importance_sampling_ratio/min": 0.9628451426823934, + "sampling/sampling_logp_difference/max": 0.0634835190915813, + "sampling/sampling_logp_difference/mean": 0.0011052653377798076, + "step": 24360, + "step_time": 4.531008168997747, + "student/entropy": 0.04819044756392638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.6, + "completions/max_terminated_length": 5.6, + "completions/mean_length": 1.051111137866974, + "completions/mean_terminated_length": 1.051111137866974, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05028162530312936, + "epoch": 0.9262142558766566, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.405570030212402, + "kd/policy_loss": -1.4169368207454682, + "kd/rkl_advantage_mean": 44.04669246673584, + "kd/rkl_advantage_p95": 62.03627297083537, + "kd/rkl_advantage_std": 27.303448788324992, + "kd/ssd_loss": 0.0, + "learning_rate": 7.382371928758592e-08, + "loss": -5.667747497558594, + "num_tokens": 276974173.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0499600172042847, + "sampling/importance_sampling_ratio/mean": 1.0002674380938212, + "sampling/importance_sampling_ratio/min": 0.980017485221227, + "sampling/sampling_logp_difference/max": 0.059124822805946074, + "sampling/sampling_logp_difference/mean": 0.0010422595796020081, + "step": 24390, + "step_time": 4.633605772635686, + "student/entropy": 0.05028162530312936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.1, + "completions/max_terminated_length": 8.1, + "completions/mean_length": 1.08555558125178, + "completions/mean_terminated_length": 1.08555558125178, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05336760046581427, + "epoch": 0.9273535108039342, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 38.477481842041016, + "kd/policy_loss": -4.244459219773611, + "kd/rkl_advantage_mean": 67.45276028315226, + "kd/rkl_advantage_p95": 85.08217385609944, + "kd/rkl_advantage_std": 31.77040496369203, + "kd/ssd_loss": 0.0, + "learning_rate": 7.268446436030835e-08, + "loss": -16.977838134765626, + "num_tokens": 277279113.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.047550199429194, + "sampling/importance_sampling_ratio/mean": 0.9999428908030192, + "sampling/importance_sampling_ratio/min": 0.950503530104955, + "sampling/sampling_logp_difference/max": 0.0757434461110582, + "sampling/sampling_logp_difference/mean": 0.0012042316768202, + "step": 24420, + "step_time": 4.42062399903128, + "student/entropy": 0.05336760046581427 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.7, + "completions/max_terminated_length": 7.7, + "completions/mean_length": 1.0744444727897644, + "completions/mean_terminated_length": 1.0744444727897644, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05015089443574349, + "epoch": 0.9284927657312118, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.124045372009277, + "kd/policy_loss": -3.0606062690416973, + "kd/rkl_advantage_mean": 64.73378259340922, + "kd/rkl_advantage_p95": 84.54747343858084, + "kd/rkl_advantage_std": 34.422943122684956, + "kd/ssd_loss": 0.0, + "learning_rate": 7.15452094330308e-08, + "loss": -12.242425537109375, + "num_tokens": 277590069.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0436107754707336, + "sampling/importance_sampling_ratio/mean": 1.0000280419985452, + "sampling/importance_sampling_ratio/min": 0.9529333015282949, + "sampling/sampling_logp_difference/max": 0.07371333179374535, + "sampling/sampling_logp_difference/mean": 0.0010406054973524685, + "step": 24450, + "step_time": 4.43129810046715, + "student/entropy": 0.05015089443574349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.966666666666667, + "completions/max_terminated_length": 5.966666666666667, + "completions/mean_length": 1.0547222415606181, + "completions/mean_terminated_length": 1.0547222415606181, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04954607623318831, + "epoch": 0.9296320206584894, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.136602401733398, + "kd/policy_loss": -0.8676694989204407, + "kd/rkl_advantage_mean": 44.41076687177022, + "kd/rkl_advantage_p95": 61.77774744828542, + "kd/rkl_advantage_std": 28.281566910942395, + "kd/ssd_loss": 0.0, + "learning_rate": 7.040595450575323e-08, + "loss": -3.470677693684896, + "num_tokens": 277902850.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0290534655253092, + "sampling/importance_sampling_ratio/mean": 1.000110963980357, + "sampling/importance_sampling_ratio/min": 0.9712700545787811, + "sampling/sampling_logp_difference/max": 0.05138137205503881, + "sampling/sampling_logp_difference/mean": 0.001023512194903257, + "step": 24480, + "step_time": 4.466964628934026, + "student/entropy": 0.04954607623318831 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.233333333333333, + "completions/max_terminated_length": 5.233333333333333, + "completions/mean_length": 1.044444465637207, + "completions/mean_terminated_length": 1.044444465637207, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.051915683504194024, + "epoch": 0.9307712755857669, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.183622360229492, + "kd/policy_loss": -0.030345930655797323, + "kd/rkl_advantage_mean": 37.83008522987366, + "kd/rkl_advantage_p95": 56.692269738515215, + "kd/rkl_advantage_std": 27.9307813256979, + "kd/ssd_loss": 0.0, + "learning_rate": 6.926669957847567e-08, + "loss": -0.1213836669921875, + "num_tokens": 278222338.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.034332267443339, + "sampling/importance_sampling_ratio/mean": 0.9996532261371612, + "sampling/importance_sampling_ratio/min": 0.9274781068166097, + "sampling/sampling_logp_difference/max": 0.09339293891874452, + "sampling/sampling_logp_difference/mean": 0.001319175745205333, + "step": 24510, + "step_time": 4.533326573568532, + "student/entropy": 0.051915683504194024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.133333333333334, + "completions/max_terminated_length": 7.133333333333334, + "completions/mean_length": 1.0644444704055787, + "completions/mean_terminated_length": 1.0644444704055787, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05160881302629908, + "epoch": 0.9319105305130445, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.970468521118164, + "kd/policy_loss": -1.7312718510627747, + "kd/rkl_advantage_mean": 58.67013686498006, + "kd/rkl_advantage_p95": 79.27343220710755, + "kd/rkl_advantage_std": 35.17943933705489, + "kd/ssd_loss": 0.0, + "learning_rate": 6.812744465119812e-08, + "loss": -6.925089009602865, + "num_tokens": 278525410.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0149003545443216, + "sampling/importance_sampling_ratio/mean": 0.9997774620850881, + "sampling/importance_sampling_ratio/min": 0.9464803755283355, + "sampling/sampling_logp_difference/max": 0.061235378473065795, + "sampling/sampling_logp_difference/mean": 0.0008693590507997821, + "step": 24540, + "step_time": 4.427712625900555, + "student/entropy": 0.05160881302629908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.0669444680213929, + "completions/mean_terminated_length": 1.0669444680213929, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.051797817647457126, + "epoch": 0.933049785440322, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 96.91964721679688, + "kd/policy_loss": -2.116815972328186, + "kd/rkl_advantage_mean": 50.600095701217654, + "kd/rkl_advantage_p95": 67.7566350142161, + "kd/rkl_advantage_std": 28.364035905897616, + "kd/ssd_loss": 0.0, + "learning_rate": 6.698818972392055e-08, + "loss": -8.467264302571614, + "num_tokens": 278836627.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0513975421587627, + "sampling/importance_sampling_ratio/mean": 1.0001238524913787, + "sampling/importance_sampling_ratio/min": 0.9597978750864665, + "sampling/sampling_logp_difference/max": 0.0813891945562015, + "sampling/sampling_logp_difference/mean": 0.0010167930314006904, + "step": 24570, + "step_time": 4.454951275396646, + "student/entropy": 0.051797817647457126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.9, + "completions/max_terminated_length": 7.9, + "completions/mean_length": 1.0686111370722453, + "completions/mean_terminated_length": 1.0686111370722453, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05214365251983206, + "epoch": 0.9341890403675996, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.72944450378418, + "kd/policy_loss": -2.776896770795186, + "kd/rkl_advantage_mean": 67.67445777257284, + "kd/rkl_advantage_p95": 90.33871416250865, + "kd/rkl_advantage_std": 38.22513896028201, + "kd/ssd_loss": 0.0, + "learning_rate": 6.584893479664299e-08, + "loss": -11.10758565266927, + "num_tokens": 279145890.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0444311261177064, + "sampling/importance_sampling_ratio/mean": 0.9999585787455241, + "sampling/importance_sampling_ratio/min": 0.9407204230626424, + "sampling/sampling_logp_difference/max": 0.08774937532531718, + "sampling/sampling_logp_difference/mean": 0.0012417068879585712, + "step": 24600, + "step_time": 4.420951297632322, + "student/entropy": 0.05214365251983206 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.533333333333333, + "completions/max_terminated_length": 8.533333333333333, + "completions/mean_length": 1.0852778077125549, + "completions/mean_terminated_length": 1.0852778077125549, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.051446001852552097, + "epoch": 0.9353282952948772, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.420167922973633, + "kd/policy_loss": -5.0806676586469015, + "kd/rkl_advantage_mean": 76.25514686902365, + "kd/rkl_advantage_p95": 96.3916196346283, + "kd/rkl_advantage_std": 35.85770205060641, + "kd/ssd_loss": 0.0, + "learning_rate": 6.470967986936544e-08, + "loss": -20.322672526041668, + "num_tokens": 279446813.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0290647745132446, + "sampling/importance_sampling_ratio/mean": 0.9999743680159251, + "sampling/importance_sampling_ratio/min": 0.9592316905657451, + "sampling/sampling_logp_difference/max": 0.07547713701302806, + "sampling/sampling_logp_difference/mean": 0.0011674225805715346, + "step": 24630, + "step_time": 4.41243197346727, + "student/entropy": 0.051446001852552097 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.233333333333333, + "completions/max_terminated_length": 8.233333333333333, + "completions/mean_length": 1.071666697661082, + "completions/mean_terminated_length": 1.071666697661082, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048764072141299644, + "epoch": 0.9364675502221547, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.708425521850586, + "kd/policy_loss": -3.8400697191556294, + "kd/rkl_advantage_mean": 73.85025405883789, + "kd/rkl_advantage_p95": 96.04378472963968, + "kd/rkl_advantage_std": 38.740490131576856, + "kd/ssd_loss": 0.0, + "learning_rate": 6.357042494208788e-08, + "loss": -15.360281372070313, + "num_tokens": 279754863.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0239212791124979, + "sampling/importance_sampling_ratio/mean": 1.0000234882036845, + "sampling/importance_sampling_ratio/min": 0.9626144429047903, + "sampling/sampling_logp_difference/max": 0.058427164470776914, + "sampling/sampling_logp_difference/mean": 0.0009650113643147051, + "step": 24660, + "step_time": 4.443995642698428, + "student/entropy": 0.048764072141299644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.2, + "completions/max_terminated_length": 6.2, + "completions/mean_length": 1.0691666920979819, + "completions/mean_terminated_length": 1.0691666920979819, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04575751631831129, + "epoch": 0.9376068051494323, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.960562705993652, + "kd/policy_loss": -2.3015432675679524, + "kd/rkl_advantage_mean": 49.78679081598918, + "kd/rkl_advantage_p95": 67.67236296335857, + "kd/rkl_advantage_std": 28.239851880073548, + "kd/ssd_loss": 0.0, + "learning_rate": 6.243117001481031e-08, + "loss": -9.206172688802083, + "num_tokens": 280057824.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0372842311859132, + "sampling/importance_sampling_ratio/mean": 1.0000176866849264, + "sampling/importance_sampling_ratio/min": 0.9642296612262726, + "sampling/sampling_logp_difference/max": 0.06456580734811723, + "sampling/sampling_logp_difference/mean": 0.0010478615178726614, + "step": 24690, + "step_time": 4.515647494701261, + "student/entropy": 0.04575751631831129 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.033333333333333, + "completions/max_terminated_length": 7.033333333333333, + "completions/mean_length": 1.067222249507904, + "completions/mean_terminated_length": 1.067222249507904, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04537076496829589, + "epoch": 0.9387460600767098, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.100512504577637, + "kd/policy_loss": -2.006526557604472, + "kd/rkl_advantage_mean": 58.7978711605072, + "kd/rkl_advantage_p95": 79.38717730045319, + "kd/rkl_advantage_std": 34.618861703077954, + "kd/ssd_loss": 0.0, + "learning_rate": 6.129191508753276e-08, + "loss": -8.026104736328126, + "num_tokens": 280365826.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.03233935435613, + "sampling/importance_sampling_ratio/mean": 0.9998538653055827, + "sampling/importance_sampling_ratio/min": 0.9407571117083232, + "sampling/sampling_logp_difference/max": 0.08446583972933391, + "sampling/sampling_logp_difference/mean": 0.0011638163899381955, + "step": 24720, + "step_time": 4.517195877965423, + "student/entropy": 0.04537076496829589 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.8, + "completions/max_terminated_length": 5.8, + "completions/mean_length": 1.0566666841506958, + "completions/mean_terminated_length": 1.0566666841506958, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04583372746904691, + "epoch": 0.9398853150039874, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.09550666809082, + "kd/policy_loss": -0.7583920200665791, + "kd/rkl_advantage_mean": 40.63761121431987, + "kd/rkl_advantage_p95": 56.144651905695596, + "kd/rkl_advantage_std": 25.875097267826398, + "kd/ssd_loss": 0.0, + "learning_rate": 6.015266016025519e-08, + "loss": -3.0335683186848956, + "num_tokens": 280675966.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.0070238629976909, + "sampling/importance_sampling_ratio/mean": 0.9999167521794637, + "sampling/importance_sampling_ratio/min": 0.9768604159355163, + "sampling/sampling_logp_difference/max": 0.041261476340393226, + "sampling/sampling_logp_difference/mean": 0.0007553001467992241, + "step": 24750, + "step_time": 4.532232762062146, + "student/entropy": 0.04583372746904691 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.8, + "completions/max_terminated_length": 4.8, + "completions/mean_length": 1.0450000206629435, + "completions/mean_terminated_length": 1.0450000206629435, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04863089028125008, + "epoch": 0.9410245699312649, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.99091148376465, + "kd/policy_loss": 0.38925105333328247, + "kd/rkl_advantage_mean": 30.19339369138082, + "kd/rkl_advantage_p95": 44.82427624861399, + "kd/rkl_advantage_std": 22.62632118066152, + "kd/ssd_loss": 0.0, + "learning_rate": 5.901340523297763e-08, + "loss": 1.557004165649414, + "num_tokens": 280984928.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0194938460985818, + "sampling/importance_sampling_ratio/mean": 1.000118378798167, + "sampling/importance_sampling_ratio/min": 0.9826298316319784, + "sampling/sampling_logp_difference/max": 0.03200130326052507, + "sampling/sampling_logp_difference/mean": 0.0007128645588333408, + "step": 24780, + "step_time": 4.540351938434954, + "student/entropy": 0.04863089028125008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.1, + "completions/max_terminated_length": 7.1, + "completions/mean_length": 1.0700000286102296, + "completions/mean_terminated_length": 1.0700000286102296, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05098900341739257, + "epoch": 0.9421638248585426, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 36.09462356567383, + "kd/policy_loss": -3.015657595793406, + "kd/rkl_advantage_mean": 60.30119067827861, + "kd/rkl_advantage_p95": 79.02205692132314, + "kd/rkl_advantage_std": 32.34345366706451, + "kd/ssd_loss": 0.0, + "learning_rate": 5.7874150305700074e-08, + "loss": -12.06262919108073, + "num_tokens": 281291612.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0222268263498941, + "sampling/importance_sampling_ratio/mean": 0.9997126917044322, + "sampling/importance_sampling_ratio/min": 0.9400642871856689, + "sampling/sampling_logp_difference/max": 0.07733054844041666, + "sampling/sampling_logp_difference/mean": 0.0011249140584065269, + "step": 24810, + "step_time": 4.435728562032939, + "student/entropy": 0.05098900341739257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.366666666666666, + "completions/max_terminated_length": 7.366666666666666, + "completions/mean_length": 1.0619444688161215, + "completions/mean_terminated_length": 1.0619444688161215, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04947279635816813, + "epoch": 0.9433030797858201, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.827193260192871, + "kd/policy_loss": -2.4090285817782084, + "kd/rkl_advantage_mean": 63.860173018773395, + "kd/rkl_advantage_p95": 85.05100040435791, + "kd/rkl_advantage_std": 36.49974303841591, + "kd/ssd_loss": 0.0, + "learning_rate": 5.6734895378422505e-08, + "loss": -9.636113484700521, + "num_tokens": 281604563.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.0255268772443136, + "sampling/importance_sampling_ratio/mean": 0.9999263266722361, + "sampling/importance_sampling_ratio/min": 0.9531001150608063, + "sampling/sampling_logp_difference/max": 0.07475586649185667, + "sampling/sampling_logp_difference/mean": 0.0009921087252829845, + "step": 24840, + "step_time": 4.448061752833504, + "student/entropy": 0.04947279635816813 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.133333333333334, + "completions/max_terminated_length": 7.133333333333334, + "completions/mean_length": 1.0777778069178263, + "completions/mean_terminated_length": 1.0777778069178263, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04932990819215775, + "epoch": 0.9444423347130977, + "eval/gt_acc_batch": 0.0019444445768992105, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.231063842773438, + "kd/policy_loss": -3.3358731627464295, + "kd/rkl_advantage_mean": 61.15788985888163, + "kd/rkl_advantage_p95": 79.23358878294627, + "kd/rkl_advantage_std": 31.437716155250868, + "kd/ssd_loss": 0.0, + "learning_rate": 5.559564045114495e-08, + "loss": -13.343495686848959, + "num_tokens": 281922939.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445148110389, + "rewards/gt_logging_reward/std": 0.017397598922252656, + "sampling/importance_sampling_ratio/max": 1.0331990679105123, + "sampling/importance_sampling_ratio/mean": 0.9999691426753998, + "sampling/importance_sampling_ratio/min": 0.9504413028558095, + "sampling/sampling_logp_difference/max": 0.06752446196042002, + "sampling/sampling_logp_difference/mean": 0.0010923027361665542, + "step": 24870, + "step_time": 4.6115738462031, + "student/entropy": 0.04932990819215775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.1, + "completions/max_terminated_length": 7.1, + "completions/mean_length": 1.06944446961085, + "completions/mean_terminated_length": 1.06944446961085, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047981713494906826, + "epoch": 0.9455815896403752, + "eval/gt_acc_batch": 0.002222222338120143, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.386430740356445, + "kd/policy_loss": -3.2863041043281553, + "kd/rkl_advantage_mean": 61.26387831370036, + "kd/rkl_advantage_p95": 79.0872527440389, + "kd/rkl_advantage_std": 31.828412792086603, + "kd/ssd_loss": 0.0, + "learning_rate": 5.445638552386739e-08, + "loss": -13.145218912760416, + "num_tokens": 282243909.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.002222222338120143, + "rewards/gt_logging_reward/std": 0.024343226353327432, + "sampling/importance_sampling_ratio/max": 1.0199476798375449, + "sampling/importance_sampling_ratio/mean": 0.9998489022254944, + "sampling/importance_sampling_ratio/min": 0.9585173646608989, + "sampling/sampling_logp_difference/max": 0.05922598019242287, + "sampling/sampling_logp_difference/mean": 0.0009683814792272945, + "step": 24900, + "step_time": 4.529212789533873, + "student/entropy": 0.047981713494906826 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.9, + "completions/max_terminated_length": 5.9, + "completions/mean_length": 1.0566666920979817, + "completions/mean_terminated_length": 1.0566666920979817, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04629517144834002, + "epoch": 0.9467208445676527, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.441429138183594, + "kd/policy_loss": -1.1566502690315246, + "kd/rkl_advantage_mean": 44.93296839396159, + "kd/rkl_advantage_p95": 61.859106246630354, + "kd/rkl_advantage_std": 28.073242682218552, + "kd/ssd_loss": 0.0, + "learning_rate": 5.331713059658983e-08, + "loss": -4.626600646972657, + "num_tokens": 282536377.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0432476997375488, + "sampling/importance_sampling_ratio/mean": 1.0000193655490874, + "sampling/importance_sampling_ratio/min": 0.9593884726365407, + "sampling/sampling_logp_difference/max": 0.05762675900477916, + "sampling/sampling_logp_difference/mean": 0.001006886786490213, + "step": 24930, + "step_time": 4.393209163368253, + "student/entropy": 0.04629517144834002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.0583333532015482, + "completions/mean_terminated_length": 1.0583333532015482, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047758288433154425, + "epoch": 0.9478600994949303, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.911986351013184, + "kd/policy_loss": -1.4447898745536805, + "kd/rkl_advantage_mean": 58.4728480497996, + "kd/rkl_advantage_p95": 79.43130962053935, + "kd/rkl_advantage_std": 35.42144641280174, + "kd/ssd_loss": 0.0, + "learning_rate": 5.217787566931227e-08, + "loss": -5.779155985514323, + "num_tokens": 282844363.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0199171503384907, + "sampling/importance_sampling_ratio/mean": 0.9997784614562988, + "sampling/importance_sampling_ratio/min": 0.9440604130427043, + "sampling/sampling_logp_difference/max": 0.06624667597934604, + "sampling/sampling_logp_difference/mean": 0.0010223019819629068, + "step": 24960, + "step_time": 4.46125135840072, + "student/entropy": 0.047758288433154425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.3, + "completions/max_terminated_length": 7.3, + "completions/mean_length": 1.0605555812517802, + "completions/mean_terminated_length": 1.0605555812517802, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047300922622283297, + "epoch": 0.9489993544222078, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.66077709197998, + "kd/policy_loss": -2.424164338906606, + "kd/rkl_advantage_mean": 62.813410981496176, + "kd/rkl_advantage_p95": 84.50054641564687, + "kd/rkl_advantage_std": 36.276329157749814, + "kd/ssd_loss": 0.0, + "learning_rate": 5.1038620742034706e-08, + "loss": -9.696659342447917, + "num_tokens": 283144325.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.055546263853709, + "sampling/importance_sampling_ratio/mean": 1.0003454625606536, + "sampling/importance_sampling_ratio/min": 0.9703868011633555, + "sampling/sampling_logp_difference/max": 0.0703469274720798, + "sampling/sampling_logp_difference/mean": 0.0010277767762696991, + "step": 24990, + "step_time": 4.384266725530809, + "student/entropy": 0.047300922622283297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.0788889169692992, + "completions/mean_terminated_length": 1.0788889169692992, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047142289609958726, + "epoch": 0.9501386093494855, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.063851356506348, + "kd/policy_loss": -3.7331292470296225, + "kd/rkl_advantage_mean": 65.73133718172708, + "kd/rkl_advantage_p95": 84.75149281819661, + "kd/rkl_advantage_std": 33.53579897085826, + "kd/ssd_loss": 0.0, + "learning_rate": 4.989936581475715e-08, + "loss": -14.932515462239584, + "num_tokens": 283449721.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0171839594841003, + "sampling/importance_sampling_ratio/mean": 0.9996907154719035, + "sampling/importance_sampling_ratio/min": 0.9393717626730601, + "sampling/sampling_logp_difference/max": 0.05789358232480784, + "sampling/sampling_logp_difference/mean": 0.0009665837036057686, + "step": 25020, + "step_time": 4.410322233933645, + "student/entropy": 0.047142289609958726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00027777779226501785, + "completions/max_length": 22.333333333333332, + "completions/max_terminated_length": 5.8, + "completions/mean_length": 1.184166701634725, + "completions/mean_terminated_length": 1.042257261276245, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04668881210188071, + "epoch": 0.951277864276763, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.600628852844238, + "kd/policy_loss": -0.7550794899463653, + "kd/rkl_advantage_mean": 47.42581613858541, + "kd/rkl_advantage_p95": 67.2501618305842, + "kd/rkl_advantage_std": 30.395278585950532, + "kd/ssd_loss": 0.0, + "learning_rate": 4.876011088747958e-08, + "loss": -3.020318603515625, + "num_tokens": 283748928.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0709010362625122, + "sampling/importance_sampling_ratio/mean": 1.0004100620746612, + "sampling/importance_sampling_ratio/min": 0.9693600376447041, + "sampling/sampling_logp_difference/max": 0.06049291348705689, + "sampling/sampling_logp_difference/mean": 0.0008292646671179682, + "step": 25050, + "step_time": 4.438078894200347, + "student/entropy": 0.04668881210188071 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.566666666666666, + "completions/max_terminated_length": 8.566666666666666, + "completions/mean_length": 1.0938889185587566, + "completions/mean_terminated_length": 1.0938889185587566, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04897224452967445, + "epoch": 0.9524171192040406, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.165634155273438, + "kd/policy_loss": -3.7763996481895448, + "kd/rkl_advantage_mean": 74.15920287768046, + "kd/rkl_advantage_p95": 96.42879486878714, + "kd/rkl_advantage_std": 38.54545406301816, + "kd/ssd_loss": 0.0, + "learning_rate": 4.7620855960202025e-08, + "loss": -15.105598958333333, + "num_tokens": 284055074.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.013867163658142, + "sampling/importance_sampling_ratio/mean": 0.9994344393412272, + "sampling/importance_sampling_ratio/min": 0.9240334113438924, + "sampling/sampling_logp_difference/max": 0.08473426311587294, + "sampling/sampling_logp_difference/mean": 0.001262340908093999, + "step": 25080, + "step_time": 4.451544605966289, + "student/entropy": 0.04897224452967445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.733333333333333, + "completions/max_terminated_length": 5.733333333333333, + "completions/mean_length": 1.0505555788675944, + "completions/mean_terminated_length": 1.0505555788675944, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.045784951529155175, + "epoch": 0.9535563741313181, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.49165916442871, + "kd/policy_loss": -1.1807011445363362, + "kd/rkl_advantage_mean": 45.08580282529195, + "kd/rkl_advantage_p95": 61.982100971539815, + "kd/rkl_advantage_std": 28.00454257329305, + "kd/ssd_loss": 0.0, + "learning_rate": 4.648160103292446e-08, + "loss": -4.722804768880208, + "num_tokens": 284359800.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0341588536898294, + "sampling/importance_sampling_ratio/mean": 0.9999333202838898, + "sampling/importance_sampling_ratio/min": 0.9486240824063619, + "sampling/sampling_logp_difference/max": 0.06261429611283044, + "sampling/sampling_logp_difference/mean": 0.0009754694163954506, + "step": 25110, + "step_time": 4.367524961100814, + "student/entropy": 0.045784951529155175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.166666666666667, + "completions/max_terminated_length": 6.166666666666667, + "completions/mean_length": 1.0511111299196878, + "completions/mean_terminated_length": 1.0511111299196878, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04597410773858428, + "epoch": 0.9546956290585957, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.80007553100586, + "kd/policy_loss": -0.8475074688593547, + "kd/rkl_advantage_mean": 48.18970855077108, + "kd/rkl_advantage_p95": 67.470254834493, + "kd/rkl_advantage_std": 31.22137617667516, + "kd/ssd_loss": 0.0, + "learning_rate": 4.534234610564691e-08, + "loss": -3.390028889973958, + "num_tokens": 284659256.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0414723674456279, + "sampling/importance_sampling_ratio/mean": 1.0001850525538127, + "sampling/importance_sampling_ratio/min": 0.9738982756932576, + "sampling/sampling_logp_difference/max": 0.06958155971951782, + "sampling/sampling_logp_difference/mean": 0.0009866078102883573, + "step": 25140, + "step_time": 4.370591018230091, + "student/entropy": 0.04597410773858428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.466666666666667, + "completions/max_terminated_length": 6.466666666666667, + "completions/mean_length": 1.0586111346880596, + "completions/mean_terminated_length": 1.0586111346880596, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048411156982183456, + "epoch": 0.9558348839858732, + "eval/gt_acc_batch": 0.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.406380653381348, + "kd/policy_loss": -1.5917023420333862, + "kd/rkl_advantage_mean": 46.84081907272339, + "kd/rkl_advantage_p95": 62.055253052711485, + "kd/rkl_advantage_std": 26.55403549820185, + "kd/ssd_loss": 0.0, + "learning_rate": 4.4203091178369344e-08, + "loss": -6.36680908203125, + "num_tokens": 284964227.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0, + "rewards/gt_logging_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.0544169704119364, + "sampling/importance_sampling_ratio/mean": 1.0002520064512888, + "sampling/importance_sampling_ratio/min": 0.9638952076435089, + "sampling/sampling_logp_difference/max": 0.0853326195385307, + "sampling/sampling_logp_difference/mean": 0.0014146221675522006, + "step": 25170, + "step_time": 4.518416389331105, + "student/entropy": 0.048411156982183456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.733333333333333, + "completions/max_terminated_length": 5.733333333333333, + "completions/mean_length": 1.0475000182787577, + "completions/mean_terminated_length": 1.0475000182787577, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.047976354137063026, + "epoch": 0.9569741389131508, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.449165344238281, + "kd/policy_loss": -0.5002148270606994, + "kd/rkl_advantage_mean": 40.605854765574136, + "kd/rkl_advantage_p95": 56.468000785509744, + "kd/rkl_advantage_std": 26.222881419459977, + "kd/ssd_loss": 0.0, + "learning_rate": 4.306383625109179e-08, + "loss": -2.0008585611979166, + "num_tokens": 285263358.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.029668664932251, + "sampling/importance_sampling_ratio/mean": 1.000099484125773, + "sampling/importance_sampling_ratio/min": 0.9715284248193105, + "sampling/sampling_logp_difference/max": 0.046518571736911936, + "sampling/sampling_logp_difference/mean": 0.000916062667965889, + "step": 25200, + "step_time": 4.485098686230291, + "student/entropy": 0.047976354137063026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.666666666666667, + "completions/max_terminated_length": 6.666666666666667, + "completions/mean_length": 1.0675000270207724, + "completions/mean_terminated_length": 1.0675000270207724, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04891963843256235, + "epoch": 0.9581133938404284, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.41252326965332, + "kd/policy_loss": -3.229990073045095, + "kd/rkl_advantage_mean": 56.31672212282817, + "kd/rkl_advantage_p95": 73.24951303799948, + "kd/rkl_advantage_std": 28.869463937481246, + "kd/ssd_loss": 0.0, + "learning_rate": 4.1924581323814226e-08, + "loss": -12.919961547851562, + "num_tokens": 285573065.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0308242400487264, + "sampling/importance_sampling_ratio/mean": 1.0000020444393158, + "sampling/importance_sampling_ratio/min": 0.9580838660399119, + "sampling/sampling_logp_difference/max": 0.06867840161236624, + "sampling/sampling_logp_difference/mean": 0.001042332467235004, + "step": 25230, + "step_time": 4.3656894378655124, + "student/entropy": 0.04891963843256235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.433333333333334, + "completions/max_terminated_length": 6.433333333333334, + "completions/mean_length": 1.067222253481547, + "completions/mean_terminated_length": 1.067222253481547, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04889163967842857, + "epoch": 0.9592526487677059, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.875031471252441, + "kd/policy_loss": -2.234720969200134, + "kd/rkl_advantage_mean": 54.21133913993835, + "kd/rkl_advantage_p95": 73.15504271189371, + "kd/rkl_advantage_std": 31.325529339909554, + "kd/ssd_loss": 0.0, + "learning_rate": 4.0785326396536663e-08, + "loss": -8.938885498046876, + "num_tokens": 285878163.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0225394209225973, + "sampling/importance_sampling_ratio/mean": 0.9997105697790781, + "sampling/importance_sampling_ratio/min": 0.9425703267256419, + "sampling/sampling_logp_difference/max": 0.06847914424724877, + "sampling/sampling_logp_difference/mean": 0.0011298666601457323, + "step": 25260, + "step_time": 4.4001869837025875, + "student/entropy": 0.04889163967842857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.4, + "completions/max_terminated_length": 7.4, + "completions/mean_length": 1.0936111450195312, + "completions/mean_terminated_length": 1.0936111450195312, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05039381692185998, + "epoch": 0.9603919036949835, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 26.833393096923828, + "kd/policy_loss": -4.931079741319021, + "kd/rkl_advantage_mean": 63.804316171010335, + "kd/rkl_advantage_p95": 79.41051065921783, + "kd/rkl_advantage_std": 27.74045608739058, + "kd/ssd_loss": 0.0, + "learning_rate": 3.96460714692591e-08, + "loss": -19.724320475260416, + "num_tokens": 286183108.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.013413905352354049, + "sampling/importance_sampling_ratio/max": 1.023403493563334, + "sampling/importance_sampling_ratio/mean": 0.9996800780296325, + "sampling/importance_sampling_ratio/min": 0.946800555785497, + "sampling/sampling_logp_difference/max": 0.06476108714317282, + "sampling/sampling_logp_difference/mean": 0.0011334112874465062, + "step": 25290, + "step_time": 4.488158798101358, + "student/entropy": 0.05039381692185998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.9, + "completions/max_terminated_length": 6.9, + "completions/mean_length": 1.075000031789144, + "completions/mean_terminated_length": 1.075000031789144, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04846996543928981, + "epoch": 0.961531158622261, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 58.33622741699219, + "kd/policy_loss": -3.124369623263677, + "kd/rkl_advantage_mean": 62.24215742746989, + "kd/rkl_advantage_p95": 84.3702641248703, + "kd/rkl_advantage_std": 35.23206140945355, + "kd/ssd_loss": 0.0, + "learning_rate": 3.850681654198154e-08, + "loss": -12.497477213541666, + "num_tokens": 286483682.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.0208491722742716, + "sampling/importance_sampling_ratio/mean": 0.9998930354913076, + "sampling/importance_sampling_ratio/min": 0.9630854249000549, + "sampling/sampling_logp_difference/max": 0.058387337733681, + "sampling/sampling_logp_difference/mean": 0.001049188309601353, + "step": 25320, + "step_time": 4.472703178766824, + "student/entropy": 0.04846996543928981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.1, + "completions/max_terminated_length": 6.1, + "completions/mean_length": 1.0722222407658895, + "completions/mean_terminated_length": 1.0722222407658895, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048111866073062025, + "epoch": 0.9626704135495386, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.84981918334961, + "kd/policy_loss": -2.1602563420931498, + "kd/rkl_advantage_mean": 47.22682768503825, + "kd/rkl_advantage_p95": 61.966007924079896, + "kd/rkl_advantage_std": 25.447853673497836, + "kd/ssd_loss": 0.0, + "learning_rate": 3.736756161470398e-08, + "loss": -8.641023763020833, + "num_tokens": 286796070.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.019499711940685907, + "sampling/importance_sampling_ratio/max": 1.0509810010592142, + "sampling/importance_sampling_ratio/mean": 1.0000647584597269, + "sampling/importance_sampling_ratio/min": 0.9637571354707082, + "sampling/sampling_logp_difference/max": 0.07123532874199251, + "sampling/sampling_logp_difference/mean": 0.0012110121984733268, + "step": 25350, + "step_time": 4.526795217499117, + "student/entropy": 0.048111866073062025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.766666666666667, + "completions/max_terminated_length": 7.766666666666667, + "completions/mean_length": 1.091111143430074, + "completions/mean_terminated_length": 1.091111143430074, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04964927891269326, + "epoch": 0.9638096684768162, + "eval/gt_acc_batch": 0.00027777779226501785, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.99251937866211, + "kd/policy_loss": -3.9499908367792766, + "kd/rkl_advantage_mean": 62.19731698036194, + "kd/rkl_advantage_p95": 79.32438249588013, + "kd/rkl_advantage_std": 29.962451100349426, + "kd/ssd_loss": 0.0, + "learning_rate": 3.622830668742642e-08, + "loss": -15.799966430664062, + "num_tokens": 287106662.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.00027777779226501785, + "rewards/gt_logging_reward/std": 0.003042903294165929, + "sampling/importance_sampling_ratio/max": 1.1216037074724834, + "sampling/importance_sampling_ratio/mean": 1.0008695463339488, + "sampling/importance_sampling_ratio/min": 0.9635593791802725, + "sampling/sampling_logp_difference/max": 0.11017069765366613, + "sampling/sampling_logp_difference/mean": 0.0015515094632670904, + "step": 25380, + "step_time": 4.469596930731011, + "student/entropy": 0.04964927891269326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.166666666666667, + "completions/max_terminated_length": 6.166666666666667, + "completions/mean_length": 1.060277803738912, + "completions/mean_terminated_length": 1.060277803738912, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04960029091065129, + "epoch": 0.9649489234040938, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.360085487365723, + "kd/policy_loss": -2.1015939831733705, + "kd/rkl_advantage_mean": 49.91928726832072, + "kd/rkl_advantage_p95": 66.90891617139181, + "kd/rkl_advantage_std": 28.319350764155388, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5089051760148864e-08, + "loss": -8.406377665201823, + "num_tokens": 287406551.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.033233932654063, + "sampling/importance_sampling_ratio/mean": 0.9998872558275859, + "sampling/importance_sampling_ratio/min": 0.9490702152252197, + "sampling/sampling_logp_difference/max": 0.0763838078516225, + "sampling/sampling_logp_difference/mean": 0.0012102841690648347, + "step": 25410, + "step_time": 4.440395115399345, + "student/entropy": 0.04960029091065129 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.866666666666666, + "completions/max_terminated_length": 4.866666666666666, + "completions/mean_length": 1.052222243944804, + "completions/mean_terminated_length": 1.052222243944804, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048265756325175366, + "epoch": 0.9660881783313713, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 41.25294876098633, + "kd/policy_loss": -0.6023543278376261, + "kd/rkl_advantage_mean": 32.23082176844279, + "kd/rkl_advantage_p95": 44.80176966190338, + "kd/rkl_advantage_std": 20.529136122763155, + "kd/ssd_loss": 0.0, + "learning_rate": 3.39497968328713e-08, + "loss": -2.409418741861979, + "num_tokens": 287716371.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0305413484573365, + "sampling/importance_sampling_ratio/mean": 0.9998089015483856, + "sampling/importance_sampling_ratio/min": 0.950643660624822, + "sampling/sampling_logp_difference/max": 0.07704463383803765, + "sampling/sampling_logp_difference/mean": 0.001071549957365884, + "step": 25440, + "step_time": 4.450429087269003, + "student/entropy": 0.048265756325175366 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.9, + "completions/max_terminated_length": 7.9, + "completions/mean_length": 1.081944477558136, + "completions/mean_terminated_length": 1.081944477558136, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05180166581024726, + "epoch": 0.9672274332586488, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.3282527923584, + "kd/policy_loss": -4.381060157219569, + "kd/rkl_advantage_mean": 76.0175890604655, + "kd/rkl_advantage_p95": 101.55128154754638, + "kd/rkl_advantage_std": 40.95517188509305, + "kd/ssd_loss": 0.0, + "learning_rate": 3.2810541905593746e-08, + "loss": -17.524239095052085, + "num_tokens": 288035578.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0317178249359131, + "sampling/importance_sampling_ratio/mean": 0.9996686816215515, + "sampling/importance_sampling_ratio/min": 0.9279121021429698, + "sampling/sampling_logp_difference/max": 0.09016232575910786, + "sampling/sampling_logp_difference/mean": 0.0013864617319389557, + "step": 25470, + "step_time": 4.52477505296459, + "student/entropy": 0.05180166581024726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.9, + "completions/max_terminated_length": 5.9, + "completions/mean_length": 1.0575000206629435, + "completions/mean_terminated_length": 1.0575000206629435, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0491204923329254, + "epoch": 0.9683666881859264, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.574499130249023, + "kd/policy_loss": -1.158948791027069, + "kd/rkl_advantage_mean": 42.09093011220296, + "kd/rkl_advantage_p95": 56.50811852614085, + "kd/rkl_advantage_std": 24.827302201092245, + "kd/ssd_loss": 0.0, + "learning_rate": 3.1671286978316177e-08, + "loss": -4.635794576009115, + "num_tokens": 288358561.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0070712169011433, + "sampling/importance_sampling_ratio/mean": 0.9998213072617849, + "sampling/importance_sampling_ratio/min": 0.9657035032908122, + "sampling/sampling_logp_difference/max": 0.0379757742707928, + "sampling/sampling_logp_difference/mean": 0.0007523218548158184, + "step": 25500, + "step_time": 4.525094931135633, + "student/entropy": 0.0491204923329254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.366666666666666, + "completions/max_terminated_length": 7.366666666666666, + "completions/mean_length": 1.0655555844306945, + "completions/mean_terminated_length": 1.0655555844306945, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.050518197224785885, + "epoch": 0.9695059431132039, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.871716499328613, + "kd/policy_loss": -3.2592116336027783, + "kd/rkl_advantage_mean": 63.301752678553264, + "kd/rkl_advantage_p95": 84.73279710610707, + "kd/rkl_advantage_std": 34.38589926858743, + "kd/ssd_loss": 0.0, + "learning_rate": 3.053203205103862e-08, + "loss": -13.036848958333334, + "num_tokens": 288664117.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.01645680864651998, + "sampling/importance_sampling_ratio/max": 1.019656252861023, + "sampling/importance_sampling_ratio/mean": 0.9996669729550679, + "sampling/importance_sampling_ratio/min": 0.931259427467982, + "sampling/sampling_logp_difference/max": 0.09224544373961786, + "sampling/sampling_logp_difference/mean": 0.0013509490857056031, + "step": 25530, + "step_time": 4.432030869002241, + "student/entropy": 0.050518197224785885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.5, + "completions/max_terminated_length": 5.5, + "completions/mean_length": 1.052500025431315, + "completions/mean_terminated_length": 1.052500025431315, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04852827421079079, + "epoch": 0.9706451980404815, + "eval/gt_acc_batch": 0.0025000001303851606, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.406042098999023, + "kd/policy_loss": -1.0593132674694061, + "kd/rkl_advantage_mean": 43.270747327804564, + "kd/rkl_advantage_p95": 61.829479201634726, + "kd/rkl_advantage_std": 28.33892242908478, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9392777123761058e-08, + "loss": -4.237252807617187, + "num_tokens": 288968842.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0025000001303851606, + "rewards/gt_logging_reward/std": 0.025585518529017765, + "sampling/importance_sampling_ratio/max": 1.0208788116772969, + "sampling/importance_sampling_ratio/mean": 1.0000693122545878, + "sampling/importance_sampling_ratio/min": 0.977794635295868, + "sampling/sampling_logp_difference/max": 0.05413123940428098, + "sampling/sampling_logp_difference/mean": 0.0009345519172105317, + "step": 25560, + "step_time": 4.4427119530979935, + "student/entropy": 0.04852827421079079 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.133333333333334, + "completions/max_terminated_length": 7.133333333333334, + "completions/mean_length": 1.079722253481547, + "completions/mean_terminated_length": 1.079722253481547, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.050312861986458304, + "epoch": 0.9717844529677591, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.910616874694824, + "kd/policy_loss": -2.640197378396988, + "kd/rkl_advantage_mean": 59.60240550041199, + "kd/rkl_advantage_p95": 79.06816083590189, + "kd/rkl_advantage_std": 33.09976639499267, + "kd/ssd_loss": 0.0, + "learning_rate": 2.82535221964835e-08, + "loss": -10.560791015625, + "num_tokens": 289280985.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0347540736198426, + "sampling/importance_sampling_ratio/mean": 0.9997069954872131, + "sampling/importance_sampling_ratio/min": 0.9383690893650055, + "sampling/sampling_logp_difference/max": 0.084508827344204, + "sampling/sampling_logp_difference/mean": 0.0012328256843223548, + "step": 25590, + "step_time": 4.539588482535327, + "student/entropy": 0.050312861986458304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.033333333333333, + "completions/max_terminated_length": 7.033333333333333, + "completions/mean_length": 1.0666666944821677, + "completions/mean_terminated_length": 1.0666666944821677, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05055684456601739, + "epoch": 0.9729237078950367, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 18.149471282958984, + "kd/policy_loss": -2.0078500866889955, + "kd/rkl_advantage_mean": 58.563752015431724, + "kd/rkl_advantage_p95": 78.97256906032563, + "kd/rkl_advantage_std": 34.34018260538578, + "kd/ssd_loss": 0.0, + "learning_rate": 2.711426726920594e-08, + "loss": -8.031399536132813, + "num_tokens": 289591041.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.026464839776357, + "sampling/importance_sampling_ratio/mean": 0.9999030629793803, + "sampling/importance_sampling_ratio/min": 0.9499433557192485, + "sampling/sampling_logp_difference/max": 0.07613491052761674, + "sampling/sampling_logp_difference/mean": 0.0011698554134151587, + "step": 25620, + "step_time": 4.500469063168081, + "student/entropy": 0.05055684456601739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.166666666666666, + "completions/max_terminated_length": 8.166666666666666, + "completions/mean_length": 1.0891667048136393, + "completions/mean_terminated_length": 1.0891667048136393, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04993938881283005, + "epoch": 0.9740629628223142, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.21608543395996, + "kd/policy_loss": -4.988881264130274, + "kd/rkl_advantage_mean": 77.52181517283121, + "kd/rkl_advantage_p95": 101.51454535325368, + "kd/rkl_advantage_std": 39.38969982465108, + "kd/ssd_loss": 0.0, + "learning_rate": 2.5975012341928377e-08, + "loss": -19.95552978515625, + "num_tokens": 289908026.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.074782164891561, + "sampling/importance_sampling_ratio/mean": 1.0003286719322204, + "sampling/importance_sampling_ratio/min": 0.9529167274634044, + "sampling/sampling_logp_difference/max": 0.10906513463705778, + "sampling/sampling_logp_difference/mean": 0.0014071562133419018, + "step": 25650, + "step_time": 4.541794390596139, + "student/entropy": 0.04993938881283005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 4.866666666666666, + "completions/max_terminated_length": 4.866666666666666, + "completions/mean_length": 1.0494444648424783, + "completions/mean_terminated_length": 1.0494444648424783, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049140931324412426, + "epoch": 0.9752022177495918, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.318861961364746, + "kd/policy_loss": 0.05371071100234985, + "kd/rkl_advantage_mean": 31.095206642150877, + "kd/rkl_advantage_p95": 45.01600112915039, + "kd/rkl_advantage_std": 22.121698370575906, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4835757414650815e-08, + "loss": 0.21484257380167643, + "num_tokens": 290231284.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0336401462554932, + "sampling/importance_sampling_ratio/mean": 1.0000857373078664, + "sampling/importance_sampling_ratio/min": 0.9566243271032969, + "sampling/sampling_logp_difference/max": 0.06747538998412589, + "sampling/sampling_logp_difference/mean": 0.0010745444160420448, + "step": 25680, + "step_time": 4.537546439296663, + "student/entropy": 0.049140931324412426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.8, + "completions/max_terminated_length": 7.8, + "completions/mean_length": 1.0769444743792216, + "completions/mean_terminated_length": 1.0769444743792216, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05009845833604534, + "epoch": 0.9763414726768693, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 39.775089263916016, + "kd/policy_loss": -4.427180198828379, + "kd/rkl_advantage_mean": 70.4063937664032, + "kd/rkl_advantage_p95": 90.60218748251597, + "kd/rkl_advantage_std": 34.553279586633046, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3696502487373256e-08, + "loss": -17.708719889322918, + "num_tokens": 290554401.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0365102330843607, + "sampling/importance_sampling_ratio/mean": 1.0000569939613342, + "sampling/importance_sampling_ratio/min": 0.9590622286001841, + "sampling/sampling_logp_difference/max": 0.0804588800529018, + "sampling/sampling_logp_difference/mean": 0.0013088921162610252, + "step": 25710, + "step_time": 4.644425738435045, + "student/entropy": 0.05009845833604534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.2, + "completions/max_terminated_length": 6.2, + "completions/mean_length": 1.055833359559377, + "completions/mean_terminated_length": 1.055833359559377, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04968384218712648, + "epoch": 0.9774807276041468, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.024867057800293, + "kd/policy_loss": -1.9585699001948038, + "kd/rkl_advantage_mean": 50.638511927922565, + "kd/rkl_advantage_p95": 68.05373748143514, + "kd/rkl_advantage_std": 29.015801366170248, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2557247560095696e-08, + "loss": -7.834280904134115, + "num_tokens": 290861666.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0511022845904032, + "sampling/importance_sampling_ratio/mean": 1.000293286641439, + "sampling/importance_sampling_ratio/min": 0.9731797734896342, + "sampling/sampling_logp_difference/max": 0.07164967055432499, + "sampling/sampling_logp_difference/mean": 0.0010376193677075207, + "step": 25740, + "step_time": 4.542813440231354, + "student/entropy": 0.04968384218712648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.3, + "completions/max_terminated_length": 6.3, + "completions/mean_length": 1.054444468021393, + "completions/mean_terminated_length": 1.054444468021393, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049758007563650605, + "epoch": 0.9786199825314245, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.922106742858887, + "kd/policy_loss": -0.8522098461786906, + "kd/rkl_advantage_mean": 45.078302176793414, + "kd/rkl_advantage_p95": 61.854536803563434, + "kd/rkl_advantage_std": 27.866685202717782, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1417992632818137e-08, + "loss": -3.4088414510091147, + "num_tokens": 291162518.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0354311029116312, + "sampling/importance_sampling_ratio/mean": 1.0001059075196583, + "sampling/importance_sampling_ratio/min": 0.9679457406202953, + "sampling/sampling_logp_difference/max": 0.057391593155140676, + "sampling/sampling_logp_difference/mean": 0.0009756810322869569, + "step": 25770, + "step_time": 4.478073701198931, + "student/entropy": 0.049758007563650605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.333333333333333, + "completions/max_terminated_length": 6.333333333333333, + "completions/mean_length": 1.055833351612091, + "completions/mean_terminated_length": 1.055833351612091, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05000793983538945, + "epoch": 0.979759237458702, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.235387802124023, + "kd/policy_loss": -1.1751712481180827, + "kd/rkl_advantage_mean": 48.75159611701965, + "kd/rkl_advantage_p95": 67.28671749432881, + "kd/rkl_advantage_std": 30.02679063578447, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0278737705540575e-08, + "loss": -4.700684611002604, + "num_tokens": 291466199.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.01037100205818812, + "sampling/importance_sampling_ratio/max": 1.0353818655014038, + "sampling/importance_sampling_ratio/mean": 1.0000001788139343, + "sampling/importance_sampling_ratio/min": 0.9643674393494924, + "sampling/sampling_logp_difference/max": 0.06058205927256495, + "sampling/sampling_logp_difference/mean": 0.0011233524356309013, + "step": 25800, + "step_time": 4.41721367060236, + "student/entropy": 0.05000793983538945 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.3, + "completions/max_terminated_length": 8.3, + "completions/mean_length": 1.0772222518920898, + "completions/mean_terminated_length": 1.0772222518920898, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.050784226041287184, + "epoch": 0.9808984923859796, + "eval/gt_acc_batch": 0.002222222338120143, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.716697692871094, + "kd/policy_loss": -4.1917402426401775, + "kd/rkl_advantage_mean": 77.68547439575195, + "kd/rkl_advantage_p95": 101.53782368501028, + "kd/rkl_advantage_std": 40.9342459688584, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9139482778263016e-08, + "loss": -16.766965738932292, + "num_tokens": 291772973.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.002222222338120143, + "rewards/gt_logging_reward/std": 0.022542615234851838, + "sampling/importance_sampling_ratio/max": 1.0438186764717101, + "sampling/importance_sampling_ratio/mean": 0.9999696691830953, + "sampling/importance_sampling_ratio/min": 0.9518485049406687, + "sampling/sampling_logp_difference/max": 0.0639969361325105, + "sampling/sampling_logp_difference/mean": 0.0011365193975507281, + "step": 25830, + "step_time": 4.480420821601971, + "student/entropy": 0.050784226041287184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.866666666666666, + "completions/max_terminated_length": 5.866666666666666, + "completions/mean_length": 1.061666691303253, + "completions/mean_terminated_length": 1.061666691303253, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04894356252625585, + "epoch": 0.9820377473132571, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.328874588012695, + "kd/policy_loss": -2.251263133684794, + "kd/rkl_advantage_mean": 46.21161559422811, + "kd/rkl_advantage_p95": 62.109093658129375, + "kd/rkl_advantage_std": 25.807507859667144, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8000227850985456e-08, + "loss": -9.005052693684895, + "num_tokens": 292095123.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0322443962097168, + "sampling/importance_sampling_ratio/mean": 0.9997015098730723, + "sampling/importance_sampling_ratio/min": 0.9335532854000728, + "sampling/sampling_logp_difference/max": 0.09061217429116368, + "sampling/sampling_logp_difference/mean": 0.0012783314232365228, + "step": 25860, + "step_time": 4.553933453701029, + "student/entropy": 0.04894356252625585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.333333333333333, + "completions/max_terminated_length": 7.333333333333333, + "completions/mean_length": 1.0619444688161215, + "completions/mean_terminated_length": 1.0619444688161215, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048638677472869554, + "epoch": 0.9831770022405347, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.089558601379395, + "kd/policy_loss": -2.427956136067708, + "kd/rkl_advantage_mean": 63.38492871920268, + "kd/rkl_advantage_p95": 84.63407980600992, + "kd/rkl_advantage_std": 36.346823408206305, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6860972923707897e-08, + "loss": -9.711823527018229, + "num_tokens": 292391170.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0166507363319397, + "sampling/importance_sampling_ratio/mean": 0.9997657040754954, + "sampling/importance_sampling_ratio/min": 0.9466374893983205, + "sampling/sampling_logp_difference/max": 0.06860436693144341, + "sampling/sampling_logp_difference/mean": 0.0009681303878702844, + "step": 25890, + "step_time": 4.4543179022337425, + "student/entropy": 0.048638677472869554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.833333333333333, + "completions/max_terminated_length": 6.833333333333333, + "completions/mean_length": 1.087500031789144, + "completions/mean_terminated_length": 1.087500031789144, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049483691031734146, + "epoch": 0.9843162571678122, + "eval/gt_acc_batch": 0.0016666667846341927, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.887099266052246, + "kd/policy_loss": -3.753270157178243, + "kd/rkl_advantage_mean": 57.702839056650795, + "kd/rkl_advantage_p95": 73.29076641400655, + "kd/rkl_advantage_std": 27.63532642374436, + "kd/ssd_loss": 0.0, + "learning_rate": 1.572171799643033e-08, + "loss": -15.01308085123698, + "num_tokens": 292698029.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666722546021, + "rewards/gt_logging_reward/std": 0.014354695628086726, + "sampling/importance_sampling_ratio/max": 1.046539040406545, + "sampling/importance_sampling_ratio/mean": 1.0000282526016235, + "sampling/importance_sampling_ratio/min": 0.9620709538459777, + "sampling/sampling_logp_difference/max": 0.05592103744857013, + "sampling/sampling_logp_difference/mean": 0.00120702968367065, + "step": 25920, + "step_time": 4.474044671503846, + "student/entropy": 0.049483691031734146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.066666666666666, + "completions/max_terminated_length": 6.066666666666666, + "completions/mean_length": 1.0450000166893005, + "completions/mean_terminated_length": 1.0450000166893005, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04870276773969332, + "epoch": 0.9854555120950899, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.583890914916992, + "kd/policy_loss": -0.5669395605723063, + "kd/rkl_advantage_mean": 44.60018719037374, + "kd/rkl_advantage_p95": 61.82748192151387, + "kd/rkl_advantage_std": 28.77577969133854, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4582463069152774e-08, + "loss": -2.2677576700846354, + "num_tokens": 293017583.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.01813086271286, + "sampling/importance_sampling_ratio/mean": 1.0000040670235952, + "sampling/importance_sampling_ratio/min": 0.9708061138788859, + "sampling/sampling_logp_difference/max": 0.0389699878791968, + "sampling/sampling_logp_difference/mean": 0.0007601800180661182, + "step": 25950, + "step_time": 4.532938090597357, + "student/entropy": 0.04870276773969332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.7, + "completions/max_terminated_length": 5.7, + "completions/mean_length": 1.0600000182787577, + "completions/mean_terminated_length": 1.0600000182787577, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04979968881234527, + "epoch": 0.9865947670223674, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 37.92854309082031, + "kd/policy_loss": -1.198219907283783, + "kd/rkl_advantage_mean": 41.73304640452067, + "kd/rkl_advantage_p95": 56.280099447568254, + "kd/rkl_advantage_std": 24.828946055968604, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3443208141875213e-08, + "loss": -4.79288075764974, + "num_tokens": 293315727.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0462936480840048, + "sampling/importance_sampling_ratio/mean": 1.0002447386582694, + "sampling/importance_sampling_ratio/min": 0.9714060644308726, + "sampling/sampling_logp_difference/max": 0.0550809969039013, + "sampling/sampling_logp_difference/mean": 0.0010280054276032994, + "step": 25980, + "step_time": 4.508884437266291, + "student/entropy": 0.04979968881234527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.066666666666666, + "completions/max_terminated_length": 7.066666666666666, + "completions/mean_length": 1.0608333587646483, + "completions/mean_terminated_length": 1.0608333587646483, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0492727669266363, + "epoch": 0.9877340219496449, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.644086837768555, + "kd/policy_loss": -2.3263852039972943, + "kd/rkl_advantage_mean": 58.88675570487976, + "kd/rkl_advantage_p95": 78.95561037063598, + "kd/rkl_advantage_std": 33.64219895303249, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2303953214597652e-08, + "loss": -9.305539957682292, + "num_tokens": 293623034.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.029632612069448, + "sampling/importance_sampling_ratio/mean": 0.9999464531739553, + "sampling/importance_sampling_ratio/min": 0.9515039503574372, + "sampling/sampling_logp_difference/max": 0.07203741202441355, + "sampling/sampling_logp_difference/mean": 0.0010794406916829757, + "step": 26010, + "step_time": 4.58418771893048, + "student/entropy": 0.0492727669266363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.066666666666666, + "completions/max_terminated_length": 7.066666666666666, + "completions/mean_length": 1.0736111402511597, + "completions/mean_terminated_length": 1.0736111402511597, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05077148688336213, + "epoch": 0.9888732768769225, + "eval/gt_acc_batch": 0.0011111111690600714, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.268156051635742, + "kd/policy_loss": -2.5605629722277325, + "kd/rkl_advantage_mean": 59.63272164662679, + "kd/rkl_advantage_p95": 79.28638538519542, + "kd/rkl_advantage_std": 33.146682962278526, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1164698287320091e-08, + "loss": -10.242251586914062, + "num_tokens": 293930203.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0011111111690600714, + "rewards/gt_logging_reward/std": 0.012171613176663716, + "sampling/importance_sampling_ratio/max": 1.0227340420087179, + "sampling/importance_sampling_ratio/mean": 0.9996434092521668, + "sampling/importance_sampling_ratio/min": 0.9271582623322805, + "sampling/sampling_logp_difference/max": 0.08768182767865558, + "sampling/sampling_logp_difference/mean": 0.001300036827645575, + "step": 26040, + "step_time": 4.490637718230816, + "student/entropy": 0.05077148688336213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.266666666666667, + "completions/max_terminated_length": 6.266666666666667, + "completions/mean_length": 1.052500017484029, + "completions/mean_terminated_length": 1.052500017484029, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04971437510102987, + "epoch": 0.9900125318042, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.183794975280762, + "kd/policy_loss": -1.2062264402707419, + "kd/rkl_advantage_mean": 49.461396010716754, + "kd/rkl_advantage_p95": 67.62610603968302, + "kd/rkl_advantage_std": 30.597509709000587, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0025443360042532e-08, + "loss": -4.824904378255209, + "num_tokens": 294236952.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.009128709882497787, + "sampling/importance_sampling_ratio/max": 1.047704315185547, + "sampling/importance_sampling_ratio/mean": 1.0001462558905283, + "sampling/importance_sampling_ratio/min": 0.9511982321739196, + "sampling/sampling_logp_difference/max": 0.07454763073474169, + "sampling/sampling_logp_difference/mean": 0.001224726169796971, + "step": 26070, + "step_time": 4.43753209199931, + "student/entropy": 0.04971437510102987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.5, + "completions/max_terminated_length": 6.5, + "completions/mean_length": 1.0611111362775167, + "completions/mean_terminated_length": 1.0611111362775167, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04956807702158888, + "epoch": 0.9911517867314776, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.419154167175293, + "kd/policy_loss": -1.91187473932902, + "kd/rkl_advantage_mean": 50.78482637405396, + "kd/rkl_advantage_p95": 67.76189716657002, + "kd/rkl_advantage_std": 29.00770397335291, + "kd/ssd_loss": 0.0, + "learning_rate": 8.886188432764971e-09, + "loss": -7.647498575846354, + "num_tokens": 294542700.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.019499711940685907, + "sampling/importance_sampling_ratio/max": 1.021676774819692, + "sampling/importance_sampling_ratio/mean": 1.0000154296557109, + "sampling/importance_sampling_ratio/min": 0.9621683756510416, + "sampling/sampling_logp_difference/max": 0.05500246702382962, + "sampling/sampling_logp_difference/mean": 0.0008708054471450547, + "step": 26100, + "step_time": 4.476002723897303, + "student/entropy": 0.04956807702158888 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.433333333333334, + "completions/max_terminated_length": 5.433333333333334, + "completions/mean_length": 1.0447222431500753, + "completions/mean_terminated_length": 1.0447222431500753, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04937378096704682, + "epoch": 0.9922910416587551, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.540053367614746, + "kd/policy_loss": -0.4451595226923625, + "kd/rkl_advantage_mean": 39.764520740509035, + "kd/rkl_advantage_p95": 56.453127479553224, + "kd/rkl_advantage_std": 26.769851966698965, + "kd/ssd_loss": 0.0, + "learning_rate": 7.74693350548741e-09, + "loss": -1.7806378682454427, + "num_tokens": 294853309.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.0214089751243591, + "sampling/importance_sampling_ratio/mean": 1.0000356038411458, + "sampling/importance_sampling_ratio/min": 0.97422975897789, + "sampling/sampling_logp_difference/max": 0.037766075149799384, + "sampling/sampling_logp_difference/mean": 0.0007910876971436665, + "step": 26130, + "step_time": 4.532526334136492, + "student/entropy": 0.04937378096704682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.333333333333334, + "completions/max_terminated_length": 8.333333333333334, + "completions/mean_length": 1.0755555868148803, + "completions/mean_terminated_length": 1.0755555868148803, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.05089043875535329, + "epoch": 0.9934302965860328, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.199850082397461, + "kd/policy_loss": -4.25898738304774, + "kd/rkl_advantage_mean": 77.89862354596455, + "kd/rkl_advantage_p95": 101.70894380410512, + "kd/rkl_advantage_std": 40.937318273385365, + "kd/ssd_loss": 0.0, + "learning_rate": 6.607678578209851e-09, + "loss": -17.03594970703125, + "num_tokens": 295166093.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.005869448184967, + "sampling/importance_sampling_ratio/mean": 0.9996325174967448, + "sampling/importance_sampling_ratio/min": 0.9438994606335958, + "sampling/sampling_logp_difference/max": 0.07309055617079138, + "sampling/sampling_logp_difference/mean": 0.0012583131591478983, + "step": 26160, + "step_time": 4.55060838979601, + "student/entropy": 0.05089043875535329 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.133333333333334, + "completions/max_terminated_length": 7.133333333333334, + "completions/mean_length": 1.065000025431315, + "completions/mean_terminated_length": 1.065000025431315, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04803143463407954, + "epoch": 0.9945695515133103, + "eval/gt_acc_batch": 0.0019444445458551248, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.278257369995117, + "kd/policy_loss": -2.391254464785258, + "kd/rkl_advantage_mean": 59.66200437545776, + "kd/rkl_advantage_p95": 79.33362863858541, + "kd/rkl_advantage_std": 33.617793376247086, + "kd/ssd_loss": 0.0, + "learning_rate": 5.4684236509322896e-09, + "loss": -9.565018717447916, + "num_tokens": 295469719.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0019444445458551248, + "rewards/gt_logging_reward/std": 0.021300323059161505, + "sampling/importance_sampling_ratio/max": 1.0238885720570883, + "sampling/importance_sampling_ratio/mean": 0.9998938461144765, + "sampling/importance_sampling_ratio/min": 0.9560864925384521, + "sampling/sampling_logp_difference/max": 0.06474281249878307, + "sampling/sampling_logp_difference/mean": 0.0009786157674777011, + "step": 26190, + "step_time": 4.515940680233083, + "student/entropy": 0.04803143463407954 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.566666666666666, + "completions/max_terminated_length": 6.566666666666666, + "completions/mean_length": 1.0550000190734863, + "completions/mean_terminated_length": 1.0550000190734863, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.04876919183880091, + "epoch": 0.9957088064405879, + "eval/gt_acc_batch": 0.0013888889613250892, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.062623977661133, + "kd/policy_loss": -0.6500392079353332, + "kd/rkl_advantage_mean": 49.02277800242106, + "kd/rkl_advantage_p95": 67.84590619405111, + "kd/rkl_advantage_std": 31.60169760088126, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3291687236547296e-09, + "loss": -2.6001571655273437, + "num_tokens": 295780005.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0013888889613250892, + "rewards/gt_logging_reward/std": 0.015214516470829645, + "sampling/importance_sampling_ratio/max": 1.0155099193255106, + "sampling/importance_sampling_ratio/mean": 1.000007293621699, + "sampling/importance_sampling_ratio/min": 0.9696869790554047, + "sampling/sampling_logp_difference/max": 0.04150601282405356, + "sampling/sampling_logp_difference/mean": 0.000764289375122947, + "step": 26220, + "step_time": 4.497556301834508, + "student/entropy": 0.04876919183880091 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.533333333333333, + "completions/max_terminated_length": 5.533333333333333, + "completions/mean_length": 1.0572222431500753, + "completions/mean_terminated_length": 1.0572222431500753, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.048992157851656275, + "epoch": 0.9968480613678654, + "eval/gt_acc_batch": 0.0005555555845300357, + "frac_reward_zero_std": 1.0, + "grad_norm": 12.799737930297852, + "kd/policy_loss": -0.7410255471865336, + "kd/rkl_advantage_mean": 37.15859508514404, + "kd/rkl_advantage_p95": 50.640459179878235, + "kd/rkl_advantage_std": 22.739436098436514, + "kd/ssd_loss": 0.0, + "learning_rate": 3.189913796377169e-09, + "loss": -2.9641019185384114, + "num_tokens": 296091155.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0005555555845300357, + "rewards/gt_logging_reward/std": 0.006085806588331858, + "sampling/importance_sampling_ratio/max": 1.035975162188212, + "sampling/importance_sampling_ratio/mean": 1.0002053439617158, + "sampling/importance_sampling_ratio/min": 0.9776621202627818, + "sampling/sampling_logp_difference/max": 0.05033983583562076, + "sampling/sampling_logp_difference/mean": 0.00091565958961534, + "step": 26250, + "step_time": 4.631865675768737, + "student/entropy": 0.048992157851656275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 5.666666666666667, + "completions/max_terminated_length": 5.666666666666667, + "completions/mean_length": 1.0472222407658895, + "completions/mean_terminated_length": 1.0472222407658895, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.0483944958075881, + "epoch": 0.9979873162951429, + "eval/gt_acc_batch": 0.001666666753590107, + "frac_reward_zero_std": 1.0, + "grad_norm": 37.7700080871582, + "kd/policy_loss": -1.0608094811439515, + "kd/rkl_advantage_mean": 41.28356536229452, + "kd/rkl_advantage_p95": 55.93631745974223, + "kd/rkl_advantage_std": 24.9317034025987, + "kd/ssd_loss": 0.0, + "learning_rate": 2.050658869099609e-09, + "loss": -4.243235015869141, + "num_tokens": 296390029.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.001666666753590107, + "rewards/gt_logging_reward/std": 0.018257419764995574, + "sampling/importance_sampling_ratio/max": 1.0120067278544107, + "sampling/importance_sampling_ratio/mean": 0.9997288604577382, + "sampling/importance_sampling_ratio/min": 0.9443143904209137, + "sampling/sampling_logp_difference/max": 0.058127597362423934, + "sampling/sampling_logp_difference/mean": 0.0008980661756747091, + "step": 26280, + "step_time": 4.508118474869601, + "student/entropy": 0.0483944958075881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 6.333333333333333, + "completions/max_terminated_length": 6.333333333333333, + "completions/mean_length": 1.0688889145851135, + "completions/mean_terminated_length": 1.0688889145851135, + "completions/min_length": 1.0, + "completions/min_terminated_length": 1.0, + "entropy": 0.049233150823662676, + "epoch": 0.9991265712224205, + "eval/gt_acc_batch": 0.0008333333767950535, + "frac_reward_zero_std": 1.0, + "grad_norm": 9.815135955810547, + "kd/policy_loss": -1.877562149365743, + "kd/rkl_advantage_mean": 50.92659360567729, + "kd/rkl_advantage_p95": 68.34898257255554, + "kd/rkl_advantage_std": 29.29863789578279, + "kd/ssd_loss": 0.0, + "learning_rate": 9.114039418220484e-10, + "loss": -7.510247802734375, + "num_tokens": 296709685.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.0008333333767950535, + "rewards/gt_logging_reward/std": 0.007328098764022192, + "sampling/importance_sampling_ratio/max": 1.0156861066818237, + "sampling/importance_sampling_ratio/mean": 0.999770446618398, + "sampling/importance_sampling_ratio/min": 0.9435775975386301, + "sampling/sampling_logp_difference/max": 0.07692603406806788, + "sampling/sampling_logp_difference/mean": 0.0011109722652084505, + "step": 26310, + "step_time": 4.531068226866288, + "student/entropy": 0.049233150823662676 + } + ], + "logging_steps": 30, + "max_steps": 26333, + "num_input_tokens_seen": 296935972, + "num_train_epochs": 1, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 10, + "trial_name": null, + "trial_params": null +}